This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 299aeec07d0 branch-3.1: [fix](parser) parse string wrong when set
NO_BACKSLASH_ESCAPES sql mode #53449 (#53542)
299aeec07d0 is described below
commit 299aeec07d0d8d790c0cde64811f56c13f1e8165
Author: morrySnow <[email protected]>
AuthorDate: Fri Jul 18 19:10:02 2025 +0800
branch-3.1: [fix](parser) parse string wrong when set NO_BACKSLASH_ESCAPES
sql mode #53449 (#53542)
cherry picked from #53449
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 9 ++--
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 ++
.../doris/nereids/parser/LogicalPlanBuilder.java | 6 +++
.../apache/doris/nereids/parser/NereidsParser.java | 4 +-
.../doris/nereids/parser/NereidsParserTest.java | 60 ++++++++++++++++++++++
.../expression/SimplifyArithmeticRuleTest.java | 4 +-
.../rules/expression/SimplifyRangeTest.java | 2 +-
.../SimplifyArithmeticComparisonRuleTest.java | 4 +-
.../doris/nereids/rules/rewrite/OrToInTest.java | 2 +-
9 files changed, 84 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 7419b51e717..a127880a5ff 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -24,6 +24,7 @@ lexer grammar DorisLexer;
* When true, parser should throw ParseExcetion for unclosed bracketed
comment.
*/
public boolean has_unclosed_bracketed_comment = false;
+ public boolean isNoBackslashEscapes = false;
/**
* Verify whether current token is a valid decimal token (which contains
dot).
@@ -609,10 +610,10 @@ ATSIGN: '@';
DOUBLEATSIGN: '@@';
STRING_LITERAL
- : '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''
- | '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'
- | 'R\'' (~'\'')* '\''
- | 'R"'(~'"')* '"'
+ : {!isNoBackslashEscapes}? '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''
+ | {isNoBackslashEscapes}? '\'' ('\'\'' | ~('\''))* '\''
+ | {!isNoBackslashEscapes}?'"' ( '\\'. | '""' | ~('"'| '\\'))* '"'
+ | {isNoBackslashEscapes}?'"' ('""' | ~('"'))* '"'
;
LEADING_STRING
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index def42b80df2..5d95771127d 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -33,6 +33,10 @@ singleStatement
: SEMICOLON* statement? SEMICOLON* EOF
;
+expressionWithEof
+ : expression EOF
+ ;
+
statement
: statementBase # statementBaseAlias
| CALL name=multipartIdentifier LEFT_PAREN (expression (COMMA
expression)*)? RIGHT_PAREN #callProcedure
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 727d99d4890..9713077d019 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -103,6 +103,7 @@ import
org.apache.doris.nereids.DorisParser.ElementAtContext;
import org.apache.doris.nereids.DorisParser.ExistContext;
import org.apache.doris.nereids.DorisParser.ExplainContext;
import org.apache.doris.nereids.DorisParser.ExportContext;
+import org.apache.doris.nereids.DorisParser.ExpressionWithEofContext;
import org.apache.doris.nereids.DorisParser.FixedPartitionDefContext;
import org.apache.doris.nereids.DorisParser.FromClauseContext;
import org.apache.doris.nereids.DorisParser.GroupingElementContext;
@@ -550,6 +551,11 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return ParserUtils.withOrigin(ctx, () -> (LogicalPlan)
visit(ctx.statement()));
}
+ @Override
+ public Expression visitExpressionWithEof(ExpressionWithEofContext ctx) {
+ return ParserUtils.withOrigin(ctx, () -> typedVisit(ctx.expression()));
+ }
+
@Override
public LogicalPlan visitStatementDefault(StatementDefaultContext ctx) {
LogicalPlan plan = plan(ctx.query());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
index c273f50b04a..f9ac5b724ef 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
@@ -36,6 +36,7 @@ import org.apache.doris.plugin.DialectConverterPlugin;
import org.apache.doris.plugin.PluginMgr;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.SqlModeHelper;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -289,7 +290,7 @@ public class NereidsParser {
if (isSimpleIdentifier(expression)) {
return new UnboundSlot(expression);
}
- return parse(expression, DorisParser::expression);
+ return parse(expression, DorisParser::expressionWithEof);
}
private static boolean isSimpleIdentifier(String expression) {
@@ -443,6 +444,7 @@ public class NereidsParser {
private static CommonTokenStream parseAllTokens(String sql) {
DorisLexer lexer = new DorisLexer(new
CaseInsensitiveStream(CharStreams.fromString(sql)));
+ lexer.isNoBackslashEscapes = SqlModeHelper.hasNoBackSlashEscapes();
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
return tokenStream;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index e7b3349c558..853414ac6f5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -27,6 +27,7 @@ import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -44,10 +45,13 @@ import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SqlModeHelper;
import org.apache.doris.qe.StmtExecutor;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import java.util.List;
import java.util.Optional;
@@ -696,4 +700,60 @@ public class NereidsParserTest extends ParserTestBase {
Assertions.fail(ex);
}
}
+
+ @Test
+ public void testNoBackSlashEscapes() {
+ testNoBackSlashEscapes("''", "", "");
+ testNoBackSlashEscapes("\"\"", "", "");
+
+ testNoBackSlashEscapes("''''", "'", "'");
+ testNoBackSlashEscapes("\"\"\"\"", "\"", "\"");
+
+ testNoBackSlashEscapes("\"\\\\n\"", "\\\\n", "\\n");
+ testNoBackSlashEscapes("\"\\t\"", "\\t", "\t");
+
+ testNoBackSlashEscapes("'\\'''", "\\'", null);
+ testNoBackSlashEscapes("'\\''", null, "'");
+ testNoBackSlashEscapes("'\\\\''", null, null);
+
+ testNoBackSlashEscapes("'\\\"\"'", "\\\"\"", "\"\"");
+ testNoBackSlashEscapes("'\\\"'", "\\\"", "\"");
+ testNoBackSlashEscapes("'\\\\\"'", "\\\\\"", "\\\"");
+
+ testNoBackSlashEscapes("\"\\''\"", "\\''", "''");
+ testNoBackSlashEscapes("\"\\'\"", "\\'", "'");
+ testNoBackSlashEscapes("\"\\\\'\"", "\\\\'", "\\'");
+
+ testNoBackSlashEscapes("\"\\\"\"\"", "\\\"", null);
+ testNoBackSlashEscapes("\"\\\"\"", null, "\"");
+ testNoBackSlashEscapes("\"\\\\\"\"", null, null);
+ }
+
+ private void testNoBackSlashEscapes(String sql, String onResult, String
offResult) {
+ NereidsParser nereidsParser = new NereidsParser();
+
+ // test on
+ try (MockedStatic<SqlModeHelper> helperMockedStatic =
Mockito.mockStatic(SqlModeHelper.class)) {
+
helperMockedStatic.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(true);
+ if (onResult == null) {
+ Assertions.assertThrowsExactly(ParseException.class, () ->
nereidsParser.parseExpression(sql),
+ "should failed when NO_BACKSLASH_ESCAPES = 1: " + sql);
+ } else {
+ Assertions.assertEquals(onResult,
+ ((StringLikeLiteral)
nereidsParser.parseExpression(sql)).getStringValue());
+ }
+ }
+
+ // test off
+ try (MockedStatic<SqlModeHelper> helperMockedStatic =
Mockito.mockStatic(SqlModeHelper.class)) {
+
helperMockedStatic.when(SqlModeHelper::hasNoBackSlashEscapes).thenReturn(false);
+ if (offResult == null) {
+ Assertions.assertThrowsExactly(ParseException.class, () ->
nereidsParser.parseExpression(sql),
+ "should failed when NO_BACKSLASH_ESCAPES = 0: " + sql);
+ } else {
+ Assertions.assertEquals(offResult,
+ ((StringLikeLiteral)
nereidsParser.parseExpression(sql)).getStringValue());
+ }
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
index f23aefe5267..de91bb7df97 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java
@@ -49,8 +49,8 @@ class SimplifyArithmeticRuleTest extends
ExpressionRewriteTestHelper {
assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as
DOUBLE) / 1.0");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as
DOUBLE) / 4.0");
assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as
DOUBLE) / 4.0");
- assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as
DOUBLE) * cast(IB as DOUBLE) / 1.0");
- assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA
as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2.0");
+ assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2", "cast(IA as
DOUBLE) * cast(IB as DOUBLE) / 1.0");
+ assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1)", "cast(IA as
DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2.0");
assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4",
"(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) /
4.0)");
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index c94a1f34894..01b8c06ff08 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -195,7 +195,7 @@ public class SimplifyRangeTest extends ExpressionRewrite {
assertRewrite("(TA + TC > 3 and TA + TC < 1) and TB < 5", "(TA + TC)
is null and null and TB < 5");
assertRewrite("(TA + TC > 3 and TA + TC < 1) or TB < 5", "((TA + TC)
is null and null) OR TB < 5");
- assertRewrite("(TA + TC > 3 OR TA < 1) AND TB = 2) AND IA =1", "(TA +
TC > 3 OR TA < 1) AND TB = 2) AND IA =1");
+ assertRewrite("(TA + TC > 3 OR TA < 1) AND TB = 2 AND IA =1", "(TA +
TC > 3 OR TA < 1) AND TB = 2 AND IA =1");
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java
index 32857d4f4ae..bc5c9ba94ca 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyArithmeticComparisonRuleTest.java
@@ -40,7 +40,7 @@ class SimplifyArithmeticComparisonRuleTest extends
ExpressionRewriteTestHelper {
assertRewriteAfterSimplify("TA - 2 > 1", "cast(TA as SMALLINT) > (1 +
2)");
assertRewriteAfterSimplify("1 + TA > 2", "cast(TA as SMALLINT) > (2 -
1)");
assertRewriteAfterSimplify("-1 + TA > 2", "cast(TA as SMALLINT) > (2 -
(-1))");
- assertRewriteAfterSimplify("1 - TA > 2", "cast(TA as SMALLINT) < (1 -
2))");
+ assertRewriteAfterSimplify("1 - TA > 2", "cast(TA as SMALLINT) < (1 -
2)");
assertRewriteAfterSimplify("-1 - TA > 2", "cast(TA as SMALLINT) <
((-1) - 2)");
assertRewriteAfterSimplify("2 * TA > 1", "((2 * TA) > 1)");
assertRewriteAfterSimplify("-2 * TA > 1", "((-2 * TA) > 1)");
@@ -72,7 +72,7 @@ class SimplifyArithmeticComparisonRuleTest extends
ExpressionRewriteTestHelper {
assertRewriteAfterSimplify("TA - 2 > 200", "cast(TA as INT) > (200 +
2)");
assertRewriteAfterSimplify("1 + TA > 200", "cast(TA as INT) > (200 -
1)");
assertRewriteAfterSimplify("-1 + TA > 200", "cast(TA as INT) > (200 -
(-1))");
- assertRewriteAfterSimplify("1 - TA > 200", "cast(TA as INT) < (1 -
200))");
+ assertRewriteAfterSimplify("1 - TA > 200", "cast(TA as INT) < (1 -
200)");
assertRewriteAfterSimplify("-1 - TA > 200", "cast(TA as INT) < ((-1) -
200)");
assertRewriteAfterSimplify("2 * TA > 200", "((2 * TA) > 200)");
assertRewriteAfterSimplify("-2 * TA > 200", "((-2 * TA) > 200)");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
index 12f07dc77be..d9a84d94c37 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/OrToInTest.java
@@ -167,7 +167,7 @@ class OrToInTest extends ExpressionRewriteTestHelper {
@Test
void test13() {
// no rewrite, because of "a like 'xyz'"
- String expr = "a like 'xyz% or a=1 or a=2': no extract";
+ String expr = "a like 'xyz% or a=1 or a=2'";
Expression expression = PARSER.parseExpression(expr);
Expression rewritten =
OrToIn.EXTRACT_MODE_INSTANCE.rewriteTree(expression, context);
Assertions.assertEquals("(a like 'xyz% or a=1 or a=2')",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
