Caideyipi commented on code in PR #17936: URL: https://github.com/apache/iotdb/pull/17936#discussion_r3464929815
########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterMatcher.java: ########## @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.subscription.columnfilter.ColumnMetadata; +import org.apache.iotdb.rpc.subscription.config.TopicConfig; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class ColumnFilterMatcher { + + private static final ColumnFilterMatcher MATCH_ALL = new ColumnFilterMatcher(null, null); + + private final BoundColumnFilter boundColumnFilter; + private final Set<String> selectedColumnNames; + private final Expression expression; + + private ColumnFilterMatcher(final Set<String> selectedColumnNames, final Expression expression) { + this(null, selectedColumnNames, expression); + } + + private ColumnFilterMatcher( + final BoundColumnFilter boundColumnFilter, + final Set<String> selectedColumnNames, + final Expression expression) { + this.boundColumnFilter = boundColumnFilter; + this.selectedColumnNames = selectedColumnNames; + this.expression = expression; + } + + public static ColumnFilterMatcher matchAll() { + return MATCH_ALL; + } + + public static ColumnFilterMatcher fromBoundColumnFilter( + final BoundColumnFilter boundColumnFilter) { + if (Objects.isNull(boundColumnFilter) || boundColumnFilter.isMatchAll()) { + return matchAll(); + } + return new ColumnFilterMatcher(boundColumnFilter, null, null); + } + + public static ColumnFilterMatcher fromTopicConfig(final TopicConfig topicConfig) + throws SubscriptionException { + if (Objects.isNull(topicConfig) + || !topicConfig.isTableTopic() + || topicConfig.isColumnFilterTrivial()) { + return matchAll(); + } + return new ColumnFilterMatcher( + null, new ColumnFilterParser().parseAndValidate(topicConfig.getColumnFilter())); + } + + public static ColumnFilterMatcher ofSelectedColumnNames(final Set<String> selectedColumnNames) { + if (Objects.isNull(selectedColumnNames)) { + return matchAll(); + } + + final Set<String> normalizedColumnNames = new HashSet<>(); + selectedColumnNames.stream() + .filter(Objects::nonNull) + .map(ColumnFilterMatcher::normalize) + .forEach(normalizedColumnNames::add); + return new ColumnFilterMatcher(Collections.unmodifiableSet(normalizedColumnNames), null); + } + + public boolean isMatchAll() { + return Objects.isNull(boundColumnFilter) + && Objects.isNull(selectedColumnNames) + && Objects.isNull(expression); + } + + public boolean shouldAutoRetainTagsAtRuntime() { + return Objects.isNull(boundColumnFilter); + } + + public boolean isTimeSelected() { + if (isMatchAll()) { + return true; + } + if (Objects.nonNull(boundColumnFilter)) { + return boundColumnFilter.isTimeSelected(); + } + if (Objects.nonNull(selectedColumnNames)) { + return selectedColumnNames.contains("time"); + } + return isTimeSelected("", ""); + } + + public boolean isTimeSelected(final String databaseName, final String tableName) { + if (isMatchAll()) { + return true; + } + if (Objects.nonNull(boundColumnFilter)) { + return boundColumnFilter.isTimeSelected(databaseName, tableName); + } + if (Objects.nonNull(selectedColumnNames)) { + return selectedColumnNames.contains("time"); + } + return ColumnFilterEvaluator.evaluate( + expression, + new ColumnMetadata( + normalizeNullable(databaseName), + normalizeNullable(tableName), + "time", + TSDataType.TIMESTAMP.name(), + ColumnCategory.TIME.name())); + } + + public Map<String, Map<String, Boolean>> getTimeSelectedByTable(final String databaseName) { + if (Objects.isNull(boundColumnFilter) || boundColumnFilter.getTimeSelectedByTable().isEmpty()) { + return Collections.emptyMap(); + } + + final String normalizedDatabaseName = + Objects.nonNull(databaseName) ? normalize(databaseName) : null; + final Map<String, Map<String, Boolean>> result = new HashMap<>(); + boundColumnFilter + .getTimeSelectedByTable() + .forEach( + (tableKey, timeSelected) -> { + if (Objects.nonNull(normalizedDatabaseName) + && !Objects.equals(normalizedDatabaseName, tableKey.getDatabaseName())) { + return; + } + result + .computeIfAbsent(tableKey.getDatabaseName(), ignored -> new HashMap<>()) + .put(tableKey.getTableName(), timeSelected); + }); + if (result.isEmpty()) { + return Collections.emptyMap(); + } + + final Map<String, Map<String, Boolean>> copied = new HashMap<>(); + result.forEach( + (database, tableMap) -> copied.put(database, Collections.unmodifiableMap(tableMap))); + return Collections.unmodifiableMap(copied); + } Review Comment: Fixed. The extra local copy was removed; the local result/table maps are wrapped as unmodifiable directly. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterBaseVisitor; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterLexer; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterParser { + + private static final Pattern SINGLE_FIELD_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*"); + private static final Pattern FUNCTION_CALL_START_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*\\(.*"); + private static final Pattern UNQUOTED_COMPARISON_RIGHT_PATTERN = + Pattern.compile("(?is).*(?:!=|<>|=)\\s*[A-Za-z_][A-Za-z_0-9]*\\s*"); + + private static final BaseErrorListener ERROR_LISTENER = + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer<?, ?> recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String message, + final RecognitionException e) { + throw new ParsingException(message, e, line, charPositionInLine + 1); + } + }; + + public Expression parseAndValidate(final String rawColumnFilter) throws SubscriptionException { + try { + final Expression expression = parse(rawColumnFilter); + ColumnFilterValidator.validate(expression); + return expression; + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format("Invalid column-filter: %s", e.getMessage()), e); + } + } + + Expression parse(final String rawColumnFilter) { + if (rawColumnFilter == null || rawColumnFilter.trim().isEmpty()) { + throw new ParsingException("column-filter should not be empty", null, 1, 1); + } + validateUnsupportedSyntax(rawColumnFilter); + + final ColumnFilterLexer lexer = new ColumnFilterLexer(CharStreams.fromString(rawColumnFilter)); + final CommonTokenStream tokenStream = new CommonTokenStream(lexer); + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser parser = + new org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser(tokenStream); Review Comment: Kept the subscription facade name as `ColumnFilterParser`; the generated ANTLR parser is referenced with its fully qualified name where needed to avoid ambiguity. Renaming the grammar would create broader generated-class churn, so I left that as-is. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterBaseVisitor; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterLexer; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterParser { + + private static final Pattern SINGLE_FIELD_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*"); + private static final Pattern FUNCTION_CALL_START_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*\\(.*"); + private static final Pattern UNQUOTED_COMPARISON_RIGHT_PATTERN = + Pattern.compile("(?is).*(?:!=|<>|=)\\s*[A-Za-z_][A-Za-z_0-9]*\\s*"); + + private static final BaseErrorListener ERROR_LISTENER = + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer<?, ?> recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String message, + final RecognitionException e) { + throw new ParsingException(message, e, line, charPositionInLine + 1); + } + }; + + public Expression parseAndValidate(final String rawColumnFilter) throws SubscriptionException { + try { + final Expression expression = parse(rawColumnFilter); + ColumnFilterValidator.validate(expression); + return expression; + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format("Invalid column-filter: %s", e.getMessage()), e); Review Comment: Checked the surrounding subscription/config validation path. These parser/validator messages follow the existing subscription exception style in this area; I kept them local rather than adding a new i18n surface for the column-filter parser in this PR. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterBaseVisitor; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterLexer; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterParser { + + private static final Pattern SINGLE_FIELD_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*"); + private static final Pattern FUNCTION_CALL_START_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*\\(.*"); + private static final Pattern UNQUOTED_COMPARISON_RIGHT_PATTERN = + Pattern.compile("(?is).*(?:!=|<>|=)\\s*[A-Za-z_][A-Za-z_0-9]*\\s*"); + + private static final BaseErrorListener ERROR_LISTENER = + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer<?, ?> recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String message, + final RecognitionException e) { + throw new ParsingException(message, e, line, charPositionInLine + 1); + } + }; + + public Expression parseAndValidate(final String rawColumnFilter) throws SubscriptionException { + try { + final Expression expression = parse(rawColumnFilter); + ColumnFilterValidator.validate(expression); + return expression; + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format("Invalid column-filter: %s", e.getMessage()), e); + } + } + + Expression parse(final String rawColumnFilter) { + if (rawColumnFilter == null || rawColumnFilter.trim().isEmpty()) { + throw new ParsingException("column-filter should not be empty", null, 1, 1); + } + validateUnsupportedSyntax(rawColumnFilter); + + final ColumnFilterLexer lexer = new ColumnFilterLexer(CharStreams.fromString(rawColumnFilter)); + final CommonTokenStream tokenStream = new CommonTokenStream(lexer); + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser parser = + new org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + parser.addErrorListener(ERROR_LISTENER); + parser.setErrorHandler( + new DefaultErrorStrategy() { + @Override + public Token recoverInline(final Parser recognizer) throws RecognitionException { + if (nextTokensContext == null) { + throw new InputMismatchException(recognizer); + } + throw new InputMismatchException(recognizer, nextTokensState, nextTokensContext); + } + }); + + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + return new AstBuilder().visit(parser.columnFilter()); + } catch (final ParsingException e) { + tokenStream.seek(0); + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return new AstBuilder().visit(parser.columnFilter()); + } + } + + private static void validateUnsupportedSyntax(final String rawColumnFilter) { + final String trimmedColumnFilter = rawColumnFilter.trim(); + if ((SINGLE_FIELD_PATTERN.matcher(rawColumnFilter).matches() + && !"true".equalsIgnoreCase(trimmedColumnFilter) + && !"false".equalsIgnoreCase(trimmedColumnFilter)) + || FUNCTION_CALL_START_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected column predicate operator", null, 1, 1); + } + if (UNQUOTED_COMPARISON_RIGHT_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected string literal", null, 1, 1); + } + for (int i = 0; i < rawColumnFilter.length(); i++) { + final char ch = rawColumnFilter.charAt(i); + if (ch == '<') { + if (i + 1 < rawColumnFilter.length() && rawColumnFilter.charAt(i + 1) == '>') { + i++; + continue; + } + throw new ParsingException("unsupported comparison operator '<'", null, 1, i + 1); + } + if (ch == '>') { + throw new ParsingException("unsupported comparison operator '>'", null, 1, i + 1); + } + if (ch == '+') { + throw new ParsingException("unexpected character '+'", null, 1, i + 1); + } + } Review Comment: Fixed. The unsupported-syntax scanner now skips quoted identifiers/literals, including escaped double quotes, so `<`, `>`, and `+` inside column names or string literals are not rejected. Added a regression test. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/ColumnFilterParser.java: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.subscription.columnfilter; + +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BooleanLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNullPredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LikePredicate; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.NotExpression; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.commons.queryengine.plan.relational.sql.parser.ParsingException; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterBaseVisitor; +import org.apache.iotdb.db.relational.grammar.sql.ColumnFilterLexer; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.InputMismatchException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class ColumnFilterParser { + + private static final Pattern SINGLE_FIELD_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*"); + private static final Pattern FUNCTION_CALL_START_PATTERN = + Pattern.compile("\\s*(?:[A-Za-z_][A-Za-z_0-9]*|\"(?:\"\"|[^\"])*\")\\s*\\(.*"); + private static final Pattern UNQUOTED_COMPARISON_RIGHT_PATTERN = + Pattern.compile("(?is).*(?:!=|<>|=)\\s*[A-Za-z_][A-Za-z_0-9]*\\s*"); + + private static final BaseErrorListener ERROR_LISTENER = + new BaseErrorListener() { + @Override + public void syntaxError( + final Recognizer<?, ?> recognizer, + final Object offendingSymbol, + final int line, + final int charPositionInLine, + final String message, + final RecognitionException e) { + throw new ParsingException(message, e, line, charPositionInLine + 1); + } + }; + + public Expression parseAndValidate(final String rawColumnFilter) throws SubscriptionException { + try { + final Expression expression = parse(rawColumnFilter); + ColumnFilterValidator.validate(expression); + return expression; + } catch (final ParsingException | IllegalArgumentException e) { + throw new SubscriptionException( + String.format("Invalid column-filter: %s", e.getMessage()), e); + } + } + + Expression parse(final String rawColumnFilter) { + if (rawColumnFilter == null || rawColumnFilter.trim().isEmpty()) { + throw new ParsingException("column-filter should not be empty", null, 1, 1); + } + validateUnsupportedSyntax(rawColumnFilter); + + final ColumnFilterLexer lexer = new ColumnFilterLexer(CharStreams.fromString(rawColumnFilter)); + final CommonTokenStream tokenStream = new CommonTokenStream(lexer); + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser parser = + new org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + parser.addErrorListener(ERROR_LISTENER); + parser.setErrorHandler( + new DefaultErrorStrategy() { + @Override + public Token recoverInline(final Parser recognizer) throws RecognitionException { + if (nextTokensContext == null) { + throw new InputMismatchException(recognizer); + } + throw new InputMismatchException(recognizer, nextTokensState, nextTokensContext); + } + }); + + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + return new AstBuilder().visit(parser.columnFilter()); + } catch (final ParsingException e) { + tokenStream.seek(0); + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return new AstBuilder().visit(parser.columnFilter()); + } + } + + private static void validateUnsupportedSyntax(final String rawColumnFilter) { + final String trimmedColumnFilter = rawColumnFilter.trim(); + if ((SINGLE_FIELD_PATTERN.matcher(rawColumnFilter).matches() + && !"true".equalsIgnoreCase(trimmedColumnFilter) + && !"false".equalsIgnoreCase(trimmedColumnFilter)) + || FUNCTION_CALL_START_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected column predicate operator", null, 1, 1); + } + if (UNQUOTED_COMPARISON_RIGHT_PATTERN.matcher(rawColumnFilter).matches()) { + throw new ParsingException("expected string literal", null, 1, 1); + } + for (int i = 0; i < rawColumnFilter.length(); i++) { + final char ch = rawColumnFilter.charAt(i); + if (ch == '<') { + if (i + 1 < rawColumnFilter.length() && rawColumnFilter.charAt(i + 1) == '>') { + i++; + continue; + } + throw new ParsingException("unsupported comparison operator '<'", null, 1, i + 1); + } + if (ch == '>') { + throw new ParsingException("unsupported comparison operator '>'", null, 1, i + 1); + } + if (ch == '+') { + throw new ParsingException("unexpected character '+'", null, 1, i + 1); + } + } + } + + private static class AstBuilder extends ColumnFilterBaseVisitor<Expression> { + + @Override + public Expression visitColumnFilter( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.ColumnFilterContext + context) { + return visit(context.booleanExpression()); + } + + @Override + public Expression visitPredicateExpression( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser + .PredicateExpressionContext + context) { + return visit(context.predicate()); + } + + @Override + public Expression visitLogicalNot( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.LogicalNotContext + context) { + return new NotExpression(visit(context.booleanExpression())); + } + + @Override + public Expression visitLogicalBinary( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.LogicalBinaryContext + context) { + final Expression left = visit(context.booleanExpression(0)); + final Expression right = visit(context.booleanExpression(1)); + return Objects.nonNull(context.AND()) + ? LogicalExpression.and(left, right) + : LogicalExpression.or(left, right); + } + + @Override + public Expression visitPredicate( + final org.apache.iotdb.db.relational.grammar.sql.ColumnFilterParser.PredicateContext + context) { + if (Objects.nonNull(context.booleanValue())) { + return visit(context.booleanValue()); + } + if (Objects.nonNull(context.booleanExpression())) { + return visit(context.booleanExpression()); + } + + final Identifier field = toIdentifier(context.field()); + if (Objects.nonNull(context.comparisonOperator())) { + return new ComparisonExpression( + Objects.nonNull(context.comparisonOperator().EQ()) + ? ComparisonExpression.Operator.EQUAL + : ComparisonExpression.Operator.NOT_EQUAL, + field, + toStringLiteral(context.string(0))); + } Review Comment: `validateUnsupportedSyntax` is a pre-parser guard for clearer errors around unsupported operators/common mistakes before the ANTLR parse. It now ignores quoted text so legal quoted names/literals are not rejected by this guard. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
