This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3763638e6c8 [improvement](parser) Avoid empty identifier suffix
contexts (#67685)
3763638e6c8 is described below
commit 3763638e6c8e3ebbc1b7079266f0d97213f0278d
Author: morrySnow <[email protected]>
AuthorDate: Wed Sep 9 15:02:30 2026 +0800
[improvement](parser) Avoid empty identifier suffix contexts (#67685)
### What problem does this PR solve?
Problem Summary:
The `errorCapturingIdentifier` rule always entered a nullable helper for
valid identifiers. In the tracked SQL corpus this created 25,406 empty
CST contexts. Move the optionality to the caller and make
`errorCapturingIdentifierExtra` non-empty, while preserving the existing
unquoted-identifier error action. This avoids allocating an empty
`RealIdentContext` on the common path and keeps the grammar structure
straightforward.
This PR also removes stale `PostProcessor` references from two JMH
harnesses. Identifier post-processing already moved into grammar actions
and the class no longer exists, so those references prevented the
benchmark profile from compiling.
Add a multipart-identifier JMH workload and a direct CST regression
test. Across two JMH runs, normalized allocation for that workload
decreased from 174,368.9 to 159,004.1 B/op (-8.81%) end-to-end and from
146,097.7 to 130,741.3 B/op (-10.51%) with pre-tokenized input.
---
.../sqlparser/benchmark/IdentifierPostProcessorBenchmark.java | 8 ++++++--
.../doris/sqlparser/benchmark/LocalRulePrefixBenchmark.java | 3 ---
.../doris/sqlparser/benchmark/QueryOrganizationBenchmark.java | 3 ---
.../src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 | 7 ++-----
.../java/org/apache/doris/sqlparser/DorisSqlParserTest.java | 10 ++++++++++
5 files changed, 18 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/IdentifierPostProcessorBenchmark.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/IdentifierPostProcessorBenchmark.java
index 27f6f9ad87d..38c3ed34bb3 100644
---
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/IdentifierPostProcessorBenchmark.java
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/IdentifierPostProcessorBenchmark.java
@@ -43,7 +43,7 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-/** Measures identifier post-processing through the public facade and with
pre-tokenized input. */
+/** Measures identifier-heavy parsing through the public facade and with
pre-tokenized input. */
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
@Fork(value = 3, jvmArgsAppend = {"-Xms1g", "-Xmx1g"})
@@ -51,7 +51,7 @@ import java.util.stream.IntStream;
@Measurement(iterations = 7, time = 400, timeUnit = TimeUnit.MILLISECONDS)
@State(Scope.Thread)
public class IdentifierPostProcessorBenchmark {
- @Param({"control", "typical", "wide", "nonReserved", "quoted"})
+ @Param({"control", "typical", "wide", "multipart", "nonReserved",
"quoted"})
public String workload;
private final DorisSqlParser facade = new DorisSqlParser();
@@ -95,6 +95,10 @@ public class IdentifierPostProcessorBenchmark {
.mapToObj(index -> "c" + index + " AS alias" + index)
.collect(Collectors.joining(", "))
+ " FROM catalog.db.fact_table";
+ case "multipart":
+ return "SELECT 1 FROM " + IntStream.range(0, 64)
+ .mapToObj(index -> "catalog" + index + ".database" +
index + ".table" + index)
+ .collect(Collectors.joining(", "));
case "nonReserved":
return "SELECT action, branch, cache, catalog, connection,
engine, format, global, name "
+ "FROM aggregate AS alias WHERE action = 1";
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/LocalRulePrefixBenchmark.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/LocalRulePrefixBenchmark.java
index 4e4aa601691..8821ab6b9e5 100644
---
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/LocalRulePrefixBenchmark.java
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/LocalRulePrefixBenchmark.java
@@ -19,7 +19,6 @@ package org.apache.doris.sqlparser.benchmark;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.parser.ParseErrorListener;
-import org.apache.doris.nereids.parser.PostProcessor;
import org.apache.doris.sqlparser.DorisSqlParser;
import org.antlr.v4.runtime.CommonTokenStream;
@@ -58,7 +57,6 @@ public class LocalRulePrefixBenchmark {
public String workload;
private final DorisSqlParser facade = new DorisSqlParser();
- private final PostProcessor postProcessor = new PostProcessor();
private final ParseErrorListener errorListener = new ParseErrorListener();
private String sql;
@@ -113,7 +111,6 @@ public class LocalRulePrefixBenchmark {
private DorisParser newParser(List<Token> input) {
CommonTokenStream stream = new CommonTokenStream(new
ListTokenSource(input));
DorisParser parser = new DorisParser(stream);
- parser.addParseListener(postProcessor);
parser.removeErrorListeners();
parser.addErrorListener(errorListener);
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
diff --git
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/QueryOrganizationBenchmark.java
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/QueryOrganizationBenchmark.java
index 8a907de7abc..6136ef7b669 100644
---
a/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/QueryOrganizationBenchmark.java
+++
b/fe/fe-sql-parser-benchmark/src/main/java/org/apache/doris/sqlparser/benchmark/QueryOrganizationBenchmark.java
@@ -19,7 +19,6 @@ package org.apache.doris.sqlparser.benchmark;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.parser.ParseErrorListener;
-import org.apache.doris.nereids.parser.PostProcessor;
import org.apache.doris.sqlparser.DorisSqlParser;
import org.antlr.v4.runtime.CommonTokenStream;
@@ -58,7 +57,6 @@ public class QueryOrganizationBenchmark {
@Param({"plainSelect", "orderedSelect", "unionTail", "parenthesizedUnion",
"inlineValues"})
public String workload;
- private final PostProcessor postProcessor = new PostProcessor();
private final ParseErrorListener errorListener = new ParseErrorListener();
private DorisSqlParser facade;
@@ -104,7 +102,6 @@ public class QueryOrganizationBenchmark {
CommonTokenStream stream = new CommonTokenStream(new
ListTokenSource(tokens));
DorisParser parser = new DorisParser(stream);
parser.ansiSQLSyntax = ansi;
- parser.addParseListener(postProcessor);
parser.removeErrorListeners();
parser.addErrorListener(errorListener);
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
diff --git
a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 569baf6b66c..9de4998c436 100644
--- a/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-sql-parser/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -2216,18 +2216,15 @@ tableSnapshot
// replace identifier with errorCapturingIdentifier where the immediate follow
symbol is not an expression, otherwise
// valid expressions such as "a-b" can be recognized as an identifier
errorCapturingIdentifier
- : identifier errorCapturingIdentifierExtra
+ : identifier errorCapturingIdentifierExtra?
;
// extra left-factoring grammar
errorCapturingIdentifierExtra
: (SUBTRACT identifier)+ #errorIdent
- | #realIdent
;
finally {
- if ($ctx instanceof ErrorIdentContext) {
- reportUnquotedIdentifier((ErrorIdentContext) $ctx);
- }
+ reportUnquotedIdentifier((ErrorIdentContext) $ctx);
}
identifier
diff --git
a/fe/fe-sql-parser/src/test/java/org/apache/doris/sqlparser/DorisSqlParserTest.java
b/fe/fe-sql-parser/src/test/java/org/apache/doris/sqlparser/DorisSqlParserTest.java
index 362c0d90404..056d59c8ca1 100644
---
a/fe/fe-sql-parser/src/test/java/org/apache/doris/sqlparser/DorisSqlParserTest.java
+++
b/fe/fe-sql-parser/src/test/java/org/apache/doris/sqlparser/DorisSqlParserTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.sqlparser;
import org.apache.doris.nereids.DorisParser;
+import org.apache.doris.nereids.DorisParser.ErrorCapturingIdentifierContext;
import org.apache.doris.nereids.DorisParser.ExpressionContext;
import org.apache.doris.nereids.DorisParser.MultiStatementsContext;
import org.apache.doris.nereids.DorisParser.SingleStatementContext;
@@ -125,6 +126,15 @@ class DorisSqlParserTest {
"Possibly unquoted identifier test-tbl detected"));
}
+ @Test
+ void doesNotBuildEmptyUnquotedIdentifierSuffixContext() {
+ DorisParser generatedParser =
parser.newParser(parser.newLexer("ordinary"));
+ ErrorCapturingIdentifierContext ctx =
generatedParser.errorCapturingIdentifier();
+
+ Assertions.assertNull(ctx.errorCapturingIdentifierExtra());
+ Assertions.assertEquals(Token.EOF,
generatedParser.getCurrentToken().getType());
+ }
+
@Test
void parsesWithoutBuildingParseTree() {
DorisParser generatedParser = parser.newParser(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
