This is an automated email from the ASF dual-hosted git repository.
airborne12 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e103c10b567 [fix](fe) Reject trailing tokens in tokenize properties
(#68701)
e103c10b567 is described below
commit e103c10b567cce649b74e23bc7a2880285e6402c
Author: Jack <[email protected]>
AuthorDate: Mon Oct 12 13:35:07 2026 +0800
[fix](fe) Reject trailing tokens in tokenize properties (#68701)
### What problem does this PR solve?
`NereidsParser.parseProperties` parsed `propertyItemList` without
requiring the parser to reach the end of the input. As a result,
`tokenize` silently accepted invalid tokens after a valid property
assignment. For example:
```sql
SELECT tokenize('alpha beta', '"parser" = "unicode" trailing_tokens');
```
The ignored suffix could hide a misspelled or incorrectly assembled
property string.
### What changed?
- Add `propertyItemListWithEof` and use it for standalone property
parsing. SQL statements that embed `propertyItemList` keep their
existing grammar.
- Preserve valid single and multiple properties, including trailing
whitespace and comments.
- Reject trailing identifiers, semicolons, closing parentheses,
unexpected symbols, and assignments without a separating comma.
`tokenize` returns its existing property format error.
- Add parser and `Tokenize` unit tests, plus two error cases in the
`test_tokenize` regression suite.
---
.../apache/doris/nereids/parser/NereidsParser.java | 2 +-
.../doris/nereids/parser/NereidsParserTest.java | 20 ++++++++++++++++++++
.../expressions/functions/scalar/TokenizeTest.java | 10 ++++++++++
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 ++++
.../suites/inverted_index_p0/test_tokenize.groovy | 9 +++++++++
5 files changed, 44 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
index 539a05ef867..7a30b276da1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
@@ -337,7 +337,7 @@ public class NereidsParser {
}
public Map<String, String> parseProperties(String properties) {
- return parse(properties, DorisParser::propertyItemList);
+ return parse(properties, parser ->
parser.propertyItemListWithEof().propertyItemList());
}
private <T> T parse(String sql, Function<DorisParser, ParserRuleContext>
parseFunction) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index e9ab09d49b2..8ed37d2a37f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -99,6 +99,26 @@ import java.util.Set;
public class NereidsParserTest extends ParserTestBase {
+ @Test
+ public void testParsePropertiesRejectsTrailingTokens() {
+ NereidsParser parser = new NereidsParser();
+ for (String suffix : new String[] {" trailing_tokens", " ;", " )", "
@", " \"extra\"=\"value\""}) {
+ Assertions.assertThrows(ParseException.class,
+ () -> parser.parseProperties("\"parser\" = \"unicode\"" +
suffix), suffix);
+ }
+ }
+
+ @Test
+ public void testParseProperties() {
+ NereidsParser parser = new NereidsParser();
+ Assertions.assertEquals(ImmutableMap.of("parser", "unicode"),
+ parser.parseProperties("\"parser\" = \"unicode\""));
+ Assertions.assertEquals(ImmutableMap.of("parser", "unicode",
"support_phrase", "true"),
+ parser.parseProperties("\"parser\" = \"unicode\",
\"support_phrase\" = \"true\""));
+ Assertions.assertEquals(ImmutableMap.of("parser", "unicode"),
+ parser.parseProperties(" \"parser\" = \"unicode\" /* comment
*/ \n -- comment\n"));
+ }
+
@Test
public void testParseMultiple() {
NereidsParser nereidsParser = new NereidsParser();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TokenizeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TokenizeTest.java
index e3f17daa841..ba543312c67 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TokenizeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/scalar/TokenizeTest.java
@@ -82,4 +82,14 @@ public class TokenizeTest extends TestWithFeService {
tokenize::checkLegalityBeforeTypeCoercion);
Assertions.assertEquals("tokenize second argument must be properties
format", exception.getMessage());
}
+
+ @Test
+ public void testTokenizeRejectsTrailingTokens() {
+ Tokenize tokenize = new Tokenize(new StringLiteral("alpha beta"),
+ new StringLiteral("\"parser\" = \"unicode\" trailing_tokens"));
+
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ tokenize::checkLegalityBeforeTypeCoercion);
+ Assertions.assertEquals("tokenize second argument must be properties
format", exception.getMessage());
+ }
}
diff --git
a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 0558f042abe..df7db1d9c7a 100644
--- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -101,6 +101,10 @@ expressionWithEof
: expression EOF
;
+propertyItemListWithEof
+ : propertyItemList EOF
+ ;
+
statement
: statementBase # statementBaseAlias
| CALL name=multipartIdentifier LEFT_PAREN (expression (COMMA
expression)*)? RIGHT_PAREN #callProcedure
diff --git a/regression-test/suites/inverted_index_p0/test_tokenize.groovy
b/regression-test/suites/inverted_index_p0/test_tokenize.groovy
index f57022ee3b4..51df6f5eb9e 100644
--- a/regression-test/suites/inverted_index_p0/test_tokenize.groovy
+++ b/regression-test/suites/inverted_index_p0/test_tokenize.groovy
@@ -18,6 +18,15 @@
import java.sql.SQLException
suite("test_tokenize"){
+ test {
+ sql """SELECT tokenize('alpha beta', '"parser" = "unicode"
trailing_tokens')"""
+ exception "tokenize second argument must be properties format"
+ }
+ test {
+ sql """SELECT tokenize('alpha beta', '"parser" = "unicode";')"""
+ exception "tokenize second argument must be properties format"
+ }
+
// prepare test table
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
