This is an automated email from the ASF dual-hosted git repository. jt2594838 pushed a commit to branch delete_with_attribute in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4f31e9f2c83467042ec001d87151ffcb2877a568 Author: Tian Jiang <[email protected]> AuthorDate: Wed Jul 1 12:08:55 2026 +0800 add test --- .../relational/it/db/it/IoTDBDeletionTableIT.java | 53 +++++++++++--- .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../iotdb/db/i18n/DataNodeQueryMessages.java | 2 + .../db/queryengine/plan/analyze/AnalyzeUtils.java | 50 ++++++++++++- .../dataregion/modification/TagPredicate.java | 84 +++++++++++++++++++++- 5 files changed, 179 insertions(+), 12 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index e146af7e53e..022bb416dd6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -505,14 +505,7 @@ public class IoTDBDeletionTableIT { } assertEquals(List.of("d1,2,12", "d2,1,21", "d4,1,41", "d4,2,42"), actual); - try { - statement.execute("DELETE FROM delete_by_attr_other_ops WHERE attr1 IS NOT NULL"); - fail("should not reach here!"); - } catch (SQLException e) { - assertEquals( - "701: Unsupported expression: (attr1 IS NOT NULL) in (attr1 IS NOT NULL)", - e.getMessage()); - } + statement.execute("DELETE FROM delete_by_attr_other_ops WHERE attr1 IS NOT NULL"); actual = new ArrayList<>(); try (ResultSet resultSet = @@ -523,7 +516,49 @@ public class IoTDBDeletionTableIT { resultSet.getString(1) + "," + resultSet.getLong(2) + "," + resultSet.getInt(3)); } } - assertEquals(List.of("d1,2,12", "d2,1,21", "d4,1,41", "d4,2,42"), actual); + assertEquals(List.of("d4,1,41", "d4,2,42"), actual); + } + } + + @Test + public void testDeleteDataByTagIsNotNullFilter() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("use test"); + statement.execute( + "CREATE TABLE delete_by_tag_not_null(" + + "deviceId STRING TAG, site STRING TAG, attr1 ATTRIBUTE, s1 INT32 FIELD)"); + statement.execute( + "INSERT INTO delete_by_tag_not_null(time, deviceId, site, attr1, s1) VALUES " + + "(1, 'd1', 'north', 'red', 11)," + + "(2, 'd1', 'north', 'red', 12)," + + "(1, 'd2', 'south', 'blue', 21)," + + "(2, 'd2', 'south', 'blue', 22)"); + statement.execute( + "INSERT INTO delete_by_tag_not_null(time, deviceId, attr1, s1) VALUES " + + "(1, 'd3', 'green', 31)," + + "(2, 'd3', 'green', 32)"); + + statement.execute("DELETE FROM delete_by_tag_not_null WHERE site IS NOT NULL AND time = 1"); + + final List<String> actual = new ArrayList<>(); + try (ResultSet resultSet = + statement.executeQuery( + "SELECT deviceId, site, time, s1 FROM delete_by_tag_not_null " + + "ORDER BY deviceId, time")) { + while (resultSet.next()) { + actual.add( + resultSet.getString(1) + + "," + + resultSet.getString(2) + + "," + + resultSet.getLong(3) + + "," + + resultSet.getInt(4)); + } + } + assertEquals( + List.of("d1,north,2,12", "d2,south,2,22", "d3,null,1,31", "d3,null,2,32"), actual); } } diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 9f8c4dd50b8..85dceb76a9a 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -271,6 +271,8 @@ public final class DataNodeQueryMessages { "The table '%s' does not contain a time column"; public static final String THE_OPERATOR_OF_TAG_PREDICATE_MUST_BE_FOR = "The operator of tag predicate must be '=' for "; + public static final String THE_COLUMN_S_DOES_NOT_EXIST_OR_IS_NOT_A_TAG_COLUMN = + "The column '%s' does not exist or is not a tag column"; public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION = "Too many devices (%d) matched by attribute filters in deletion, limit is %d. " + "Please remove all attribute filters (%s) or add more attribute filters."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 6d61d2bef70..a015779237b 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -269,6 +269,8 @@ public final class DataNodeQueryMessages { "表 '%s' 不包含时间列"; public static final String THE_OPERATOR_OF_TAG_PREDICATE_MUST_BE_FOR = "标签谓词的运算符必须为 '='，目标："; + public static final String THE_COLUMN_S_DOES_NOT_EXIST_OR_IS_NOT_A_TAG_COLUMN = + "列 '%s' 不存在或不是标签列"; public static final String TOO_MANY_DEVICES_MATCHED_BY_ATTRIBUTE_FILTERS_IN_DELETION = "删除语句中的属性过滤条件匹配了过多设备（%d），限制为 %d。请移除所有属性过滤条件（%s），或进一步增加属性过滤条件。"; public static final String ONLY_TIME_FILTERS_ARE_SUPPORTED_IN_LAST_QUERY = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 598535b70a8..6557cb97142 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNotNullPredicate; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression.Operator; @@ -66,6 +67,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.DeviceIn; import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentNotNull; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -522,6 +524,20 @@ public class AnalyzeUtils { } deviceFilterExpressions.add(toSymbolReferenceExpression(currExp)); } + } else if (currExp instanceof IsNotNullPredicate) { + final PredicateParseResult parseResult = + parseIsNotNull((IsNotNullPredicate) currExp, tagPredicate, table); + tagPredicate = parseResult.tagPredicate; + if (parseResult.shouldQueryDevice()) { + if (Objects.isNull(deviceFilterExpressions)) { + deviceFilterExpressions = new ArrayList<>(); + } + if (Objects.isNull(attributeColumns)) { + attributeColumns = new ArrayList<>(); + } + deviceFilterExpressions.add(toSymbolReferenceExpression(currExp)); + collectAttributeColumn(attributeColumns, parseResult.attributeColumn); + } } else { throw new SemanticException( DataNodeQueryMessages.UNSUPPORTED_EXPRESSION + currExp + " in " + expression); @@ -595,7 +611,9 @@ public class AnalyzeUtils { int tagColumnOrdinal = table.getTagColumnOrdinal(columnName); if (tagColumnOrdinal == -1) { throw new SemanticException( - "The column '" + columnName + "' does not exist or is not a tag column"); + String.format( + DataNodeQueryMessages.THE_COLUMN_S_DOES_NOT_EXIST_OR_IS_NOT_A_TAG_COLUMN, + columnName)); } // the first segment is the table name, so + 1 @@ -603,6 +621,32 @@ public class AnalyzeUtils { return PredicateParseResult.tag(combinePredicates(oldPredicate, newPredicate)); } + private static PredicateParseResult parseIsNotNull( + IsNotNullPredicate isNotNullPredicate, TagPredicate oldPredicate, TsTable table) { + Expression leftHandExp = isNotNullPredicate.getValue(); + if (!(leftHandExp instanceof Identifier)) { + throw new SemanticException( + DataNodeQueryMessages.LEFT_HAND_EXPRESSION_IS_NOT_AN_IDENTIFIER + leftHandExp); + } + String columnName = ((Identifier) leftHandExp).getValue(); + final TsTableColumnSchema columnSchema = table.getColumnSchema(columnName); + if (Objects.nonNull(columnSchema) + && columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { + return PredicateParseResult.attribute(columnName, oldPredicate); + } + int tagColumnOrdinal = table.getTagColumnOrdinal(columnName); + if (tagColumnOrdinal == -1) { + throw new SemanticException( + String.format( + DataNodeQueryMessages.THE_COLUMN_S_DOES_NOT_EXIST_OR_IS_NOT_A_TAG_COLUMN, + columnName)); + } + + // the first segment is the table name, so + 1 + TagPredicate newPredicate = new SegmentNotNull(tagColumnOrdinal + 1); + return PredicateParseResult.tag(combinePredicates(oldPredicate, newPredicate)); + } + private static TagPredicate combinePredicates( TagPredicate oldPredicate, TagPredicate newPredicate) { if (oldPredicate == null) { @@ -673,7 +717,9 @@ public class AnalyzeUtils { int tagColumnOrdinal = table.getTagColumnOrdinal(columnName); if (tagColumnOrdinal == -1) { throw new SemanticException( - "The column '" + columnName + "' does not exist or is not a tag column"); + String.format( + DataNodeQueryMessages.THE_COLUMN_S_DOES_NOT_EXIST_OR_IS_NOT_A_TAG_COLUMN, + columnName)); } TagPredicate newPredicate = getTagPredicate(comparisonExpression, right, tagColumnOrdinal); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java index 3a6f0a6ec35..8d41d23ca06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java @@ -55,7 +55,8 @@ public abstract class TagPredicate implements StreamSerializable, BufferSerializ FULL_EXACT_MATCH, SEGMENT_EXACT_MATCH, AND, - DEVICE_IN; + DEVICE_IN, + SEGMENT_NOT_NULL; public long serialize(OutputStream stream) throws IOException { stream.write((byte) ordinal()); @@ -111,6 +112,8 @@ public abstract class TagPredicate implements StreamSerializable, BufferSerializ predicate = new And(); } else if (Objects.requireNonNull(type) == TagPredicateType.DEVICE_IN) { predicate = new DeviceIn(); + } else if (Objects.requireNonNull(type) == TagPredicateType.SEGMENT_NOT_NULL) { + predicate = new SegmentNotNull(); } else { throw new IllegalArgumentException(StorageEngineMessages.UNRECOGNIZED_PREDICATE_TYPE + type); } @@ -131,6 +134,8 @@ public abstract class TagPredicate implements StreamSerializable, BufferSerializ predicate = new And(); } else if (Objects.requireNonNull(type) == TagPredicateType.DEVICE_IN) { predicate = new DeviceIn(); + } else if (Objects.requireNonNull(type) == TagPredicateType.SEGMENT_NOT_NULL) { + predicate = new SegmentNotNull(); } else { throw new IllegalArgumentException(StorageEngineMessages.UNRECOGNIZED_PREDICATE_TYPE + type); } @@ -442,6 +447,83 @@ public abstract class TagPredicate implements StreamSerializable, BufferSerializ } } + public static class SegmentNotNull extends TagPredicate { + + public static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(SegmentNotNull.class); + private int segmentIndex; + + public SegmentNotNull(int segmentIndex) { + super(TagPredicateType.SEGMENT_NOT_NULL); + this.segmentIndex = segmentIndex; + } + + public SegmentNotNull() { + super(TagPredicateType.SEGMENT_NOT_NULL); + } + + @Override + public int serializedSize() { + return super.serializedSize() + ReadWriteForEncodingUtils.varIntSize(segmentIndex); + } + + @Override + public long serialize(OutputStream stream) throws IOException { + long size = super.serialize(stream); + size += ReadWriteForEncodingUtils.writeVarInt(segmentIndex, stream); + return size; + } + + @Override + public long serialize(ByteBuffer buffer) { + long size = super.serialize(buffer); + size += ReadWriteForEncodingUtils.writeVarInt(segmentIndex, buffer); + return size; + } + + @Override + public void deserialize(InputStream stream) throws IOException { + segmentIndex = ReadWriteForEncodingUtils.readVarInt(stream); + } + + @Override + public void deserialize(ByteBuffer buffer) { + segmentIndex = ReadWriteForEncodingUtils.readVarInt(buffer); + } + + @Override + public boolean matches(IDeviceID deviceID) { + return deviceID.segment(segmentIndex) != null; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SegmentNotNull that = (SegmentNotNull) o; + return segmentIndex == that.segmentIndex; + } + + @Override + public int hashCode() { + return Objects.hashCode(segmentIndex); + } + + @Override + public String toString() { + return "SegmentNotNull{" + "segmentIndex=" + segmentIndex + '}'; + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE; + } + } + public static class And extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(And.class);
