Caideyipi commented on code in PR #17936: URL: https://github.com/apache/iotdb/pull/17936#discussion_r3464930608
########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterValidator.java: ########## @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.CommonQueryAstVisitor; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; + +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public class ColumnFilterValidator implements CommonQueryAstVisitor<Void, Void> { + + private static final Set<String> LEGAL_FIELDS = + Set.of("database", "table_name", "column_name", "datatype", "category"); + private static final String REGEXP_LIKE = "regexp_like"; + + public static void validate(final Expression expression) { + new ColumnFilterValidator().process(expression); + } + + @Override + public Void visitNode(final Node node, final Void context) { + throw invalid("unsupported expression: " + node.getClass().getSimpleName()); + } + + @Override + public Void visitBooleanLiteral(final BooleanLiteral node, final Void context) { + return null; + } + + @Override + public Void visitLogicalExpression(final LogicalExpression node, final Void context) { + node.getTerms().forEach(this::process); + return null; + } + + @Override + public Void visitNotExpression(final NotExpression node, final Void context) { + process(node.getValue()); + return null; + } + + @Override + public Void visitComparisonExpression(final ComparisonExpression node, final Void context) { + if (node.getOperator() != ComparisonExpression.Operator.EQUAL + && node.getOperator() != ComparisonExpression.Operator.NOT_EQUAL) { + throw invalid("only =, !=, and <> comparisons are supported in column-filter"); + } + requireField(node.getLeft()); + requireStringLiteral(node.getRight(), "comparison right operand"); + return null; + } + + @Override + public Void visitInPredicate(final InPredicate node, final Void context) { + requireField(node.getValue()); + if (!(node.getValueList() instanceof InListExpression)) { + throw invalid("IN predicate must use a string literal list"); + } + for (final Expression expression : ((InListExpression) node.getValueList()).getValues()) { + requireStringLiteral(expression, "IN element"); + } + return null; + } + + @Override + public Void visitLikePredicate(final LikePredicate node, final Void context) { + requireField(node.getValue()); + final StringLiteral pattern = requireStringLiteral(node.getPattern(), "LIKE pattern"); + final String escape = + node.getEscape() + .map(expression -> requireStringLiteral(expression, "LIKE escape").getValue()) + .orElse(null); + ColumnFilterEvaluator.compileLikePattern(pattern.getValue(), escape); + return null; + } + + @Override + public Void visitFunctionCall(final FunctionCall node, final Void context) { + if (!REGEXP_LIKE.equalsIgnoreCase(node.getName().toString()) + || node.isDistinct() + || node.getProcessingMode().isPresent() + || node.getArguments().size() != 2) { + throw invalid("only REGEXP is supported as regexp_like(field, pattern)"); + } Review Comment: I kept parsing and semantic validation separated. The parser builds the AST; the validator enforces the supported metadata fields, predicates, and pattern validity. Squashing them would push semantic checks into the grammar visitor and make the diagnostics harder to keep consistent. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java: ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Prunes table-model Tablets according to a subscription column matcher. */ +public class TabletColumnPruner { + + private TabletColumnPruner() { + // utility class + } + + public static Tablet pruneTableModelTablet( + final Tablet tablet, final String databaseName, final ColumnFilterMatcher matcher) { + if (Objects.isNull(tablet) || Objects.isNull(databaseName)) { + return tablet; + } + + final ColumnFilterMatcher effectiveMatcher = + Objects.nonNull(matcher) ? matcher : ColumnFilterMatcher.matchAll(); + final List<IMeasurementSchema> schemas = tablet.getSchemas(); Review Comment: Fixed. `matchAll` now returns the original tablet immediately, with test coverage. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java: ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Prunes table-model Tablets according to a subscription column matcher. */ +public class TabletColumnPruner { + + private TabletColumnPruner() { + // utility class + } + + public static Tablet pruneTableModelTablet( + final Tablet tablet, final String databaseName, final ColumnFilterMatcher matcher) { + if (Objects.isNull(tablet) || Objects.isNull(databaseName)) { + return tablet; + } + + final ColumnFilterMatcher effectiveMatcher = + Objects.nonNull(matcher) ? matcher : ColumnFilterMatcher.matchAll(); + final List<IMeasurementSchema> schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + if (Objects.isNull(schemas) || schemas.isEmpty() || Objects.isNull(values)) { + return null; + } + + final List<ColumnCategory> categories = getColumnCategories(tablet, schemas.size()); + final boolean[] selectedColumns = new boolean[schemas.size()]; + boolean hasMatchedColumn = false; + + for (int i = 0; i < schemas.size(); i++) { + if (!isValidColumn(schemas, values, i)) { + continue; + } + final IMeasurementSchema schema = schemas.get(i); + final ColumnCategory category = categories.get(i); + if (effectiveMatcher.match( + databaseName, + tablet.getTableName(), + schema.getMeasurementName(), + schema.getType(), + category)) { + hasMatchedColumn = true; + if (category != ColumnCategory.ATTRIBUTE) { + selectedColumns[i] = true; + } + } + } + + if (!hasMatchedColumn) { + return null; + } + + if (effectiveMatcher.shouldAutoRetainTagsAtRuntime()) { + for (int i = 0; i < schemas.size(); i++) { + if (isValidColumn(schemas, values, i) + && categories.get(i) == ColumnCategory.TAG + && hasValueArray(values, i)) { + selectedColumns[i] = true; + } + } + } Review Comment: I kept the tag-retention pass separate from the matched-column pass because they apply different conditions: matched non-attribute columns are selected first, then tags are auto-retained only for runtime expression matching. Keeping them separate avoids mixing those two rules. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TabletColumnPruner.java: ########## @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** Prunes table-model Tablets according to a subscription column matcher. */ +public class TabletColumnPruner { + + private TabletColumnPruner() { + // utility class + } + + public static Tablet pruneTableModelTablet( + final Tablet tablet, final String databaseName, final ColumnFilterMatcher matcher) { + if (Objects.isNull(tablet) || Objects.isNull(databaseName)) { + return tablet; + } + + final ColumnFilterMatcher effectiveMatcher = + Objects.nonNull(matcher) ? matcher : ColumnFilterMatcher.matchAll(); + final List<IMeasurementSchema> schemas = tablet.getSchemas(); + final Object[] values = tablet.getValues(); + if (Objects.isNull(schemas) || schemas.isEmpty() || Objects.isNull(values)) { + return null; + } + + final List<ColumnCategory> categories = getColumnCategories(tablet, schemas.size()); + final boolean[] selectedColumns = new boolean[schemas.size()]; + boolean hasMatchedColumn = false; + + for (int i = 0; i < schemas.size(); i++) { + if (!isValidColumn(schemas, values, i)) { + continue; + } + final IMeasurementSchema schema = schemas.get(i); + final ColumnCategory category = categories.get(i); + if (effectiveMatcher.match( + databaseName, + tablet.getTableName(), + schema.getMeasurementName(), + schema.getType(), + category)) { + hasMatchedColumn = true; + if (category != ColumnCategory.ATTRIBUTE) { + selectedColumns[i] = true; + } + } + } + + if (!hasMatchedColumn) { + return null; + } + + if (effectiveMatcher.shouldAutoRetainTagsAtRuntime()) { + for (int i = 0; i < schemas.size(); i++) { + if (isValidColumn(schemas, values, i) + && categories.get(i) == ColumnCategory.TAG + && hasValueArray(values, i)) { + selectedColumns[i] = true; + } + } + } + + final List<Integer> selectedIndices = getSelectedIndices(selectedColumns); + if (selectedIndices.isEmpty()) { + return null; + } + if (selectedIndices.size() == schemas.size()) { + return tablet; + } + + final List<IMeasurementSchema> prunedSchemas = new ArrayList<>(selectedIndices.size()); + final List<ColumnCategory> prunedCategories = new ArrayList<>(selectedIndices.size()); + final Object[] prunedValues = new Object[selectedIndices.size()]; + final BitMap[] bitMaps = tablet.getBitMaps(); + final BitMap[] prunedBitMaps = + Objects.nonNull(bitMaps) ? new BitMap[selectedIndices.size()] : null; + + for (int i = 0; i < selectedIndices.size(); i++) { + final int originalIndex = selectedIndices.get(i); + final IMeasurementSchema originalSchema = schemas.get(originalIndex); + prunedSchemas.add( + new MeasurementSchema(originalSchema.getMeasurementName(), originalSchema.getType())); + prunedCategories.add(categories.get(originalIndex)); + prunedValues[i] = values[originalIndex]; + if (Objects.nonNull(bitMaps) && originalIndex < bitMaps.length) { + prunedBitMaps[i] = bitMaps[originalIndex]; + } + } + + return new Tablet( + tablet.getTableName(), + prunedSchemas, + prunedCategories, + tablet.getTimestamps(), + prunedValues, + prunedBitMaps, + tablet.getRowSize()); + } Review Comment: Agree this could be useful as a reusable `Tablet` API. For this PR I kept pruning local to avoid changing the TsFile-facing API/dependency surface; it can be extracted once there is another caller. ########## iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterBinderTest.java: ########## @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.schema.table.TreeViewSchema; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TimeColumnSchema; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.config.TopicConstant; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ColumnFilterBinderTest { + + @Test + public void testBindSnapshotKeepsOnlyBoundTagsAtRuntime() { + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("column_name = \"temperature\""), + Collections.singletonMap( + "db", Collections.singletonMap("sensors", createTableSchema()))); + final ColumnFilterMatcher matcher = ColumnFilterMatcher.fromBoundColumnFilter(boundFilter); + + final Tablet prunedTablet = + TabletColumnPruner.pruneTableModelTablet(createRuntimeTabletWithNewTag(), "db", matcher); + + Assert.assertNotNull(prunedTablet); + Assert.assertEquals(2, prunedTablet.getSchemas().size()); + Assert.assertEquals("device", prunedTablet.getSchemas().get(0).getMeasurementName()); + Assert.assertEquals("temperature", prunedTablet.getSchemas().get(1).getMeasurementName()); + } + + @Test + public void testAttributeMatchBindsTagsButNotAttribute() { + final BoundColumnFilter boundFilter = + new ColumnFilterBinder() + .bind( + createTableTopicConfig("category = \"ATTRIBUTE\""), + Collections.singletonMap( + "db", Collections.singletonMap("sensors", createTableSchema()))); Review Comment: I kept this allowed because the filter language describes column metadata, not only output columns. Matching `ATTRIBUTE` can be useful as a predicate to select the table/retain tags, while the pruner still excludes attribute columns from emitted tablets. ########## iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/ColumnFilter.g4: ########## Review Comment: Kept the grammar name for now to avoid broad generated-parser churn. The runtime facade already avoids collision by referencing the generated parser with a fully qualified name. I can do the grammar/package rename separately if you prefer. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
