This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 577e72c [CALCITE-2993] ParseException may be thrown for legal SQL
queries due to incorrect "LOOKAHEAD(1)" hints
577e72c is described below
commit 577e72cb95e5d4d88f187909c14d8691c266ac4c
Author: Hongze Zhang <[email protected]>
AuthorDate: Sat Apr 13 04:43:44 2019 +0800
[CALCITE-2993] ParseException may be thrown for legal SQL queries due to
incorrect "LOOKAHEAD(1)" hints
Also:
* Following [CALCITE-883], add absent non-reserved keywords to config.fmpp
files;
* Remove redundant "LOOKAHEAD(1)" hints;
* The case "select json_object(key: value) from t" is added to unit test
but not fixed. See the reason from method "SqlParserTest#testJsonObject".
---
babel/src/main/codegen/config.fmpp | 2 +
core/src/main/codegen/templates/Parser.jj | 19 ++++++---
core/src/test/codegen/config.fmpp | 2 +
.../apache/calcite/sql/parser/SqlParserTest.java | 46 ++++++++++++++++++++++
piglet/src/main/javacc/PigletParser.jj | 1 -
server/src/main/codegen/config.fmpp | 2 +
6 files changed, 66 insertions(+), 6 deletions(-)
diff --git a/babel/src/main/codegen/config.fmpp
b/babel/src/main/codegen/config.fmpp
index cd5a36d..dc9f513 100644
--- a/babel/src/main/codegen/config.fmpp
+++ b/babel/src/main/codegen/config.fmpp
@@ -124,6 +124,7 @@ data: {
"GOTO"
"GRANTED"
"HIERARCHY"
+ "IGNORE"
"IMMEDIATE"
"IMMEDIATELY"
"IMPLEMENTATION"
@@ -210,6 +211,7 @@ data: {
"RELATIVE"
"REPEATABLE"
"REPLACE"
+ "RESPECT"
"RESTART"
"RESTRICT"
"RETURNED_CARDINALITY"
diff --git a/core/src/main/codegen/templates/Parser.jj
b/core/src/main/codegen/templates/Parser.jj
index cd49717..f9af1f0 100644
--- a/core/src/main/codegen/templates/Parser.jj
+++ b/core/src/main/codegen/templates/Parser.jj
@@ -1894,7 +1894,7 @@ SqlNode TableRef2(boolean lateral) :
tableRef = unnestOp.createCall(s.end(this), args.toArray());
}
|
- [ LOOKAHEAD(1) <LATERAL> { lateral = true; } ]
+ [<LATERAL> { lateral = true; } ]
<TABLE> { s = span(); } <LPAREN>
tableRef = TableFunctionCall(s.pos())
<RPAREN>
@@ -2597,7 +2597,17 @@ SqlMatchRecognize MatchRecognize(SqlNode tableRef) :
s1.end(var), var);
}
|
- [ LOOKAHEAD(1) <LAST> ] var = SimpleIdentifier() {
+ // This "LOOKAHEAD({true})" is a workaround for Babel.
+ // Because of babel parser uses option "LOOKAHEAD=2" globally,
+ // JavaCC generates something like "LOOKAHEAD(2, [<LAST>]
SimpleIdentifier())"
+ // here. But the correct LOOKAHEAD should be
+ // "LOOKAHEAD(2, [ LOOKAHEAD(2, <LAST> SimpleIdentifier())
<LAST> ]
+ // SimpleIdentifier())" which have the syntactic lookahead for
<LAST> considered.
+ //
+ // Overall LOOKAHEAD({true}) is even better as this is the
last branch in the
+ // choice.
+ LOOKAHEAD({true})
+ [ LOOKAHEAD(2, <LAST> SimpleIdentifier()) <LAST> ] var =
SimpleIdentifier() {
after = SqlMatchRecognize.SKIP_TO_LAST.createCall(
s1.end(var), var);
}
@@ -3423,7 +3433,6 @@ SqlNode AtomicRowExpression() :
}
{
(
- LOOKAHEAD(1)
e = Literal()
|
e = DynamicParam()
@@ -5221,7 +5230,7 @@ List<SqlNode> JsonNameAndValue() :
}
{
[
- LOOKAHEAD(1)
+ LOOKAHEAD(2, <KEY> JsonName())
<KEY> { kvMode = true; }
]
e = JsonName() {
@@ -5664,7 +5673,7 @@ SqlNode NamedFunctionCall() :
call = createCall(qualifiedName, s.end(this), funcType, quantifier,
args);
}
[
- LOOKAHEAD(1) call = nullTreatment(call)
+ LOOKAHEAD(2) call = nullTreatment(call)
]
[
call = withinGroup(call)
diff --git a/core/src/test/codegen/config.fmpp
b/core/src/test/codegen/config.fmpp
index 230b280..7c24366 100644
--- a/core/src/test/codegen/config.fmpp
+++ b/core/src/test/codegen/config.fmpp
@@ -128,6 +128,7 @@ data: {
"GOTO"
"GRANTED"
"HIERARCHY"
+ "IGNORE"
"IMMEDIATE"
"IMMEDIATELY"
"IMPLEMENTATION"
@@ -214,6 +215,7 @@ data: {
"RELATIVE"
"REPEATABLE"
"REPLACE"
+ "RESPECT"
"RESTART"
"RESTRICT"
"RETURNED_CARDINALITY"
diff --git
a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
index bee50f9..93c5fa8 100644
--- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -4276,6 +4276,15 @@ public class SqlParserTest {
+ "ORDER BY `COL1`\n"
+ "FETCH NEXT 10 ROWS ONLY";
sql(sql).ok(expected);
+
+ // See [CALCITE-2993] ParseException may be thrown for legal
+ // SQL queries due to incorrect "LOOKAHEAD(1)" hints
+ sql("select lead(x) ignore from t")
+ .ok("SELECT LEAD(`X`) AS `IGNORE`\n"
+ + "FROM `T`");
+ sql("select lead(x) respect from t")
+ .ok("SELECT LEAD(`X`) AS `RESPECT`\n"
+ + "FROM `T`");
}
@Test public void testAs() {
@@ -8142,6 +8151,31 @@ public class SqlParserTest {
sql(sql).ok(expected);
}
+ /** Test case for
+ * <a
href="https://issues.apache.org/jira/browse/CALCITE-2993";>[CALCITE-2993]
+ * ParseException may be thrown for legal SQL queries due to incorrect
+ * "LOOKAHEAD(1)" hints</a>. */
+ @Test public void testMatchRecognizePatternSkip6() {
+ final String sql = "select *\n"
+ + " from t match_recognize\n"
+ + " (\n"
+ + " after match skip to last\n"
+ + " pattern (strt down+ up+)\n"
+ + " define\n"
+ + " down as down.price < PREV(down.price),\n"
+ + " up as up.price > prev(up.price)\n"
+ + " ) mr";
+ final String expected = "SELECT *\n"
+ + "FROM `T` MATCH_RECOGNIZE(\n"
+ + "AFTER MATCH SKIP TO LAST `LAST`\n"
+ + "PATTERN (((`STRT` (`DOWN` +)) (`UP` +)))\n"
+ + "DEFINE "
+ + "`DOWN` AS (`DOWN`.`PRICE` < PREV(`DOWN`.`PRICE`, 1)), "
+ + "`UP` AS (`UP`.`PRICE` > PREV(`UP`.`PRICE`, 1))"
+ + ") AS `MR`";
+ sql(sql).ok(expected);
+ }
+
@Test public void testMatchRecognizeSubset1() {
final String sql = "select *\n"
+ " from t match_recognize\n"
@@ -8454,6 +8488,18 @@ public class SqlParserTest {
"JSON_OBJECT(KEY 'foo' VALUE "
+ "JSON_OBJECT(KEY 'foo' VALUE 'bar' NULL ON NULL) "
+ "FORMAT JSON NULL ON NULL)");
+
+ if (!Bug.TODO_FIXED) {
+ return;
+ }
+ // "LOOKAHEAD(2) list = JsonNameAndValue()" does not generate
+ // valid LOOKAHEAD codes for the case "key: value".
+ //
+ // You can see the generated codes that are located at method
+ // SqlParserImpl#JsonObjectFunctionCall. Looking ahead fails
+ // immediately after seeking the tokens <KEY> and <COLON>.
+ checkExp("json_object(key: value)",
+ "JSON_OBJECT(KEY `KEY` VALUE `VALUE` NULL ON NULL)");
}
@Test public void testJsonType() {
diff --git a/piglet/src/main/javacc/PigletParser.jj
b/piglet/src/main/javacc/PigletParser.jj
index 30f31e6..dbd505b 100644
--- a/piglet/src/main/javacc/PigletParser.jj
+++ b/piglet/src/main/javacc/PigletParser.jj
@@ -312,7 +312,6 @@ Assignment foreachStmt(final Identifier target) :
{
<FOREACH> id = simpleIdentifier()
(
- LOOKAHEAD(1)
<GENERATE> expList = expCommaList() <SEMICOLON> {
return new ForeachStmt(pos3(target), target, id, expList, schema);
}
diff --git a/server/src/main/codegen/config.fmpp
b/server/src/main/codegen/config.fmpp
index 821e752..296db46 100644
--- a/server/src/main/codegen/config.fmpp
+++ b/server/src/main/codegen/config.fmpp
@@ -136,6 +136,7 @@ data: {
"GOTO"
"GRANTED"
"HIERARCHY"
+ "IGNORE"
"IMMEDIATE"
"IMMEDIATELY"
"IMPLEMENTATION"
@@ -222,6 +223,7 @@ data: {
"RELATIVE"
"REPEATABLE"
"REPLACE"
+ "RESPECT"
"RESTART"
"RESTRICT"
"RETURNED_CARDINALITY"
