This is an automated email from the ASF dual-hosted git repository.
jhyde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 986a2d5 [CALCITE-2453] Parse list of SQL statements separated with a
semicolon (Chunwei Lei, charbel yazbeck)
986a2d5 is described below
commit 986a2d579c8f9b9f08aa9bbbfe11efc4e7bb0809
Author: cyazbeck <[email protected]>
AuthorDate: Wed Aug 8 12:41:13 2018 +0300
[CALCITE-2453] Parse list of SQL statements separated with a semicolon
(Chunwei Lei, charbel yazbeck)
Close apache/calcite#1177
---
core/src/main/codegen/templates/Parser.jj | 62 ++++--
.../calcite/sql/parser/SqlAbstractParserImpl.java | 10 +
.../org/apache/calcite/sql/parser/SqlParser.java | 36 +++-
.../apache/calcite/sql/parser/SqlParserTest.java | 236 ++++++++++++++++++++-
site/_docs/reference.md | 3 +
5 files changed, 312 insertions(+), 35 deletions(-)
diff --git a/core/src/main/codegen/templates/Parser.jj
b/core/src/main/codegen/templates/Parser.jj
index e83c946..6499e4b 100644
--- a/core/src/main/codegen/templates/Parser.jj
+++ b/core/src/main/codegen/templates/Parser.jj
@@ -166,8 +166,7 @@ public class ${parser.class} extends SqlAbstractParserImpl
}
};
- public SqlParseException normalizeException(Throwable ex)
- {
+ public SqlParseException normalizeException(Throwable ex) {
try {
if (ex instanceof ParseException) {
ex = cleanupParseException((ParseException) ex);
@@ -178,8 +177,7 @@ public class ${parser.class} extends SqlAbstractParserImpl
}
}
- public Metadata getMetadata()
- {
+ public Metadata getMetadata() {
synchronized (${parser.class}.class) {
if (metadata == null) {
metadata = new MetadataImpl(
@@ -189,48 +187,44 @@ public class ${parser.class} extends SqlAbstractParserImpl
}
}
- public void setTabSize(int tabSize)
- {
+ public void setTabSize(int tabSize) {
jj_input_stream.setTabSize(tabSize);
}
- public void switchTo(String stateName)
- {
+ public void switchTo(String stateName) {
int state = Arrays.asList(${parser.class}TokenManager.lexStateNames)
.indexOf(stateName);
token_source.SwitchTo(state);
}
- public void setQuotedCasing(Casing quotedCasing)
- {
+ public void setQuotedCasing(Casing quotedCasing) {
this.quotedCasing = quotedCasing;
}
- public void setUnquotedCasing(Casing unquotedCasing)
- {
+ public void setUnquotedCasing(Casing unquotedCasing) {
this.unquotedCasing = unquotedCasing;
}
- public void setIdentifierMaxLength(int identifierMaxLength)
- {
+ public void setIdentifierMaxLength(int identifierMaxLength) {
this.identifierMaxLength = identifierMaxLength;
}
- public void setConformance(SqlConformance conformance)
- {
+ public void setConformance(SqlConformance conformance) {
this.conformance = conformance;
}
- public SqlNode parseSqlExpressionEof() throws Exception
- {
+ public SqlNode parseSqlExpressionEof() throws Exception {
return SqlExpressionEof();
}
- public SqlNode parseSqlStmtEof() throws Exception
- {
+ public SqlNode parseSqlStmtEof() throws Exception {
return SqlStmtEof();
}
+ public SqlNodeList parseSqlStmtList() throws Exception {
+ return SqlStmtList();
+ }
+
private SqlNode extend(SqlNode table, SqlNodeList extendList) {
return SqlStdOperatorTable.EXTEND.createCall(
Span.of(table, extendList).pos(), table, extendList);
@@ -943,6 +937,34 @@ SqlNode SqlQueryEof() :
}
/**
+ * Parses a list of SQL statements separated by semicolon.
+ * The semicolon is required between statements, but is
+ * optional at the end.
+ */
+SqlNodeList SqlStmtList() :
+{
+ final List<SqlNode> stmtList = new ArrayList<SqlNode>();
+ SqlNode stmt;
+}
+{
+ stmt = SqlStmt() {
+ stmtList.add(stmt);
+ }
+ (
+ <SEMICOLON>
+ [
+ stmt = SqlStmt() {
+ stmtList.add(stmt);
+ }
+ ]
+ )*
+ <EOF>
+ {
+ return new SqlNodeList(stmtList, Span.of(stmtList).pos());
+ }
+}
+
+/**
* Parses an SQL statement.
*/
SqlNode SqlStmt() :
diff --git
a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java
b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java
index 4645b43..aabb9d8 100644
---
a/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java
+++
b/core/src/main/java/org/apache/calcite/sql/parser/SqlAbstractParserImpl.java
@@ -22,6 +22,7 @@ import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSyntax;
import org.apache.calcite.sql.SqlUnresolvedFunction;
@@ -451,6 +452,15 @@ public abstract class SqlAbstractParserImpl {
public abstract SqlNode parseSqlStmtEof() throws Exception;
/**
+ * Parses a list of SQL statements separated by semicolon and constructs a
+ * parse tree. The semicolon is required between statements, but is
+ * optional at the end.
+ *
+ * @return constructed list of SQL statements.
+ */
+ public abstract SqlNodeList parseSqlStmtList() throws Exception;
+
+ /**
* Sets the tab stop size.
*
* @param tabSize Tab stop size
diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParser.java
b/core/src/main/java/org/apache/calcite/sql/parser/SqlParser.java
index 50772b4..61e0a01 100644
--- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParser.java
+++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParser.java
@@ -21,6 +21,7 @@ import org.apache.calcite.avatica.util.Quoting;
import org.apache.calcite.config.Lex;
import org.apache.calcite.runtime.CalciteContextException;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.parser.impl.SqlParserImpl;
import org.apache.calcite.sql.validate.SqlConformance;
import org.apache.calcite.sql.validate.SqlConformanceEnum;
@@ -135,6 +136,17 @@ public class SqlParser {
}
}
+ /** Normalizes a SQL exception. */
+ private SqlParseException handleException(Throwable ex) {
+ if (ex instanceof CalciteContextException) {
+ final String originalSql = parser.getOriginalSql();
+ if (originalSql != null) {
+ ((CalciteContextException) ex).setOriginalStatement(originalSql);
+ }
+ }
+ return parser.normalizeException(ex);
+ }
+
/**
* Parses a <code>SELECT</code> statement.
*
@@ -147,13 +159,7 @@ public class SqlParser {
try {
return parser.parseSqlStmtEof();
} catch (Throwable ex) {
- if (ex instanceof CalciteContextException) {
- final String originalSql = parser.getOriginalSql();
- if (originalSql != null) {
- ((CalciteContextException) ex).setOriginalStatement(originalSql);
- }
- }
- throw parser.normalizeException(ex);
+ throw handleException(ex);
}
}
@@ -182,6 +188,22 @@ public class SqlParser {
}
/**
+ * Parses a list of SQL statements separated by semicolon.
+ * The semicolon is required between statements, but is
+ * optional at the end.
+ *
+ * @return list of SqlNodeList representing the list of SQL statements
+ * @throws SqlParseException if there is a parse error
+ */
+ public SqlNodeList parseStmtList() throws SqlParseException {
+ try {
+ return parser.parseSqlStmtList();
+ } catch (Throwable ex) {
+ throw handleException(ex);
+ }
+ }
+
+ /**
* Get the parser metadata.
*
* @return {@link SqlAbstractParserImpl.Metadata} implementation of
diff --git
a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
index 93c5fa8..88046f4 100644
--- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
+++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserTest.java
@@ -22,6 +22,7 @@ import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlSetOption;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.calcite.sql.parser.impl.SqlParserImpl;
@@ -45,6 +46,7 @@ import org.hamcrest.BaseMatcher;
import org.hamcrest.CustomTypeSafeMatcher;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
+import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@@ -62,6 +64,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import static org.hamcrest.CoreMatchers.equalTo;
@@ -593,6 +596,12 @@ public class SqlParserTest {
return new Sql(sql);
}
+ /** Creates an instance of helper class {@link SqlList} to test parsing a
+ * list of statements. */
+ protected SqlList sqlList(String sql) {
+ return new SqlList(sql);
+ }
+
/**
* Implementors of custom parsing logic who want to reuse this test should
* override this method with the factory for their extension parser.
@@ -1167,6 +1176,110 @@ public class SqlParserTest {
}
}
+ /** Parses a list of statements (that contains only one statement). */
+ @Test public void testStmtListWithSelect() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ sqlList("select * from emp, dept")
+ .ok(expected);
+ }
+
+ @Test public void testStmtListWithSelectAndSemicolon() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ sqlList("select * from emp, dept;")
+ .ok(expected);
+ }
+
+ @Test public void testStmtListWithTwoSelect() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ sqlList("select * from emp, dept ; select * from emp, dept")
+ .ok(expected, expected);
+ }
+
+ @Test public void testStmtListWithTwoSelectSemicolon() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ sqlList("select * from emp, dept ; select * from emp, dept;")
+ .ok(expected, expected);
+ }
+
+ @Test public void testStmtListWithSelectDelete() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ final String expected1 = "DELETE FROM `EMP`";
+ sqlList("select * from emp, dept; delete from emp")
+ .ok(expected, expected1);
+ }
+
+ @Test public void testStmtListWithSelectDeleteUpdate() {
+ final String sql = "select * from emp, dept; "
+ + "delete from emp; "
+ + "update emps set empno = empno + 1";
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ final String expected1 = "DELETE FROM `EMP`";
+ final String expected2 = "UPDATE `EMPS` SET `EMPNO` = (`EMPNO` + 1)";
+ sqlList(sql).ok(expected, expected1, expected2);
+ }
+
+ @Test public void testStmtListWithSemiColonInComment() {
+ final String sql = ""
+ + "select * from emp, dept; // comment with semicolon ; values 1\n"
+ + "values 2";
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`,\n"
+ + "`DEPT`";
+ final String expected1 = "VALUES (ROW(2))";
+ sqlList(sql).ok(expected, expected1);
+ }
+
+ @Test public void testStmtListWithSemiColonInWhere() {
+ final String expected = "SELECT *\n"
+ + "FROM `EMP`\n"
+ + "WHERE (`NAME` LIKE 'toto;')";
+ final String expected1 = "DELETE FROM `EMP`";
+ sqlList("select * from emp where name like 'toto;'; delete from emp")
+ .ok(expected, expected1);
+ }
+
+ @Test public void testStmtListWithInsertSelectInsert() {
+ final String sql = "insert into dept (name, deptno) values ('a', 123); "
+ + "select * from emp where name like 'toto;'; "
+ + "insert into dept (name, deptno) values ('b', 123);";
+ final String expected = "INSERT INTO `DEPT` (`NAME`, `DEPTNO`)\n"
+ + "VALUES (ROW('a', 123))";
+ final String expected1 = "SELECT *\n"
+ + "FROM `EMP`\n"
+ + "WHERE (`NAME` LIKE 'toto;')";
+ final String expected2 = "INSERT INTO `DEPT` (`NAME`, `DEPTNO`)\n"
+ + "VALUES (ROW('b', 123))";
+ sqlList(sql).ok(expected, expected1, expected2);
+ }
+
+ /** Should fail since the first statement lacks semicolon */
+ @Test public void testStmtListWithoutSemiColon1() {
+ sqlList("select * from emp where name like 'toto' "
+ + "^delete^ from emp")
+ .fails("(?s).*Encountered \"delete\" at .*");
+ }
+
+ /** Should fail since the third statement lacks semicolon */
+ @Test public void testStmtListWithoutSemiColon2() {
+ sqlList("select * from emp where name like 'toto'; "
+ + "delete from emp; "
+ + "insert into dept (name, deptno) values ('a', 123) "
+ + "^select^ * from dept")
+ .fails("(?s).*Encountered \"select\" at .*");
+ }
+
@Test public void testIsDistinctFrom() {
check(
"select x is distinct from y from t",
@@ -8634,11 +8747,13 @@ public class SqlParserTest {
* Callback to control how test actions are performed.
*/
protected interface Tester {
+ void checkList(String sql, List<String> expected);
+
void check(String sql, String expected);
void checkExp(String sql, String expected);
- void checkFails(String sql, String expectedMsgPattern);
+ void checkFails(String sql, boolean list, String expectedMsgPattern);
void checkExpFails(String sql, String expectedMsgPattern);
@@ -8651,11 +8766,9 @@ public class SqlParserTest {
* Default implementation of {@link Tester}.
*/
protected class TesterImpl implements Tester {
- public void check(
- String sql,
+ private void check(
+ SqlNode sqlNode,
String expected) {
- final SqlNode sqlNode = parseStmtAndHandleEx(sql);
-
// no dialect, always parenthesize
String actual = sqlNode.toSqlString(null, true).getSql();
if (LINUXIFY.get()[0]) {
@@ -8664,6 +8777,25 @@ public class SqlParserTest {
TestUtil.assertEqualsVerbose(expected, actual);
}
+ @Override public void checkList(
+ String sql,
+ List<String> expected) {
+ final SqlNodeList sqlNodeList = parseStmtsAndHandleEx(sql);
+ assertThat(sqlNodeList.size(), is(expected.size()));
+
+ for (int i = 0; i < sqlNodeList.size(); i++) {
+ SqlNode sqlNode = sqlNodeList.get(i);
+ check(sqlNode, expected.get(i));
+ }
+ }
+
+ public void check(
+ String sql,
+ String expected) {
+ final SqlNode sqlNode = parseStmtAndHandleEx(sql);
+ check(sqlNode, expected);
+ }
+
protected SqlNode parseStmtAndHandleEx(String sql) {
final SqlNode sqlNode;
try {
@@ -8674,6 +8806,17 @@ public class SqlParserTest {
return sqlNode;
}
+ /** Parses a list of statements. */
+ protected SqlNodeList parseStmtsAndHandleEx(String sql) {
+ final SqlNodeList sqlNodeList;
+ try {
+ sqlNodeList = getSqlParser(sql).parseStmtList();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("Error while parsing SQL: " + sql, e);
+ }
+ return sqlNodeList;
+ }
+
public void checkExp(
String sql,
String expected) {
@@ -8697,11 +8840,17 @@ public class SqlParserTest {
public void checkFails(
String sql,
+ boolean list,
String expectedMsgPattern) {
SqlParserUtil.StringAndPos sap = SqlParserUtil.findPos(sql);
Throwable thrown = null;
try {
- final SqlNode sqlNode = getSqlParser(sap.sql).parseStmt();
+ final SqlNode sqlNode;
+ if (list) {
+ sqlNode = getSqlParser(sap.sql).parseStmtList();
+ } else {
+ sqlNode = getSqlParser(sap.sql).parseStmt();
+ }
Util.discard(sqlNode);
} catch (Throwable ex) {
thrown = ex;
@@ -8754,6 +8903,54 @@ public class SqlParserTest {
* unparsing a query are consistent with the original query.
*/
public class UnparsingTesterImpl extends TesterImpl {
+
+ private String toSqlString(SqlNodeList sqlNodeList) {
+ List<String> sqls = sqlNodeList.getList().stream()
+ .map(it -> it.toSqlString(CalciteSqlDialect.DEFAULT, false).getSql())
+ .collect(Collectors.toList());
+ return String.join(";", sqls);
+ }
+
+ private void checkList(SqlNodeList sqlNodeList, List<String> expected) {
+ Assert.assertEquals(expected.size(), sqlNodeList.size());
+
+ for (int i = 0; i < sqlNodeList.size(); i++) {
+ SqlNode sqlNode = sqlNodeList.get(i);
+ // Unparse with no dialect, always parenthesize.
+ final String actual = sqlNode.toSqlString(null, true).getSql();
+ assertEquals(expected.get(i), linux(actual));
+ }
+ }
+
+ @Override public void checkList(String sql, List<String> expected) {
+ SqlNodeList sqlNodeList = parseStmtsAndHandleEx(sql);
+
+ checkList(sqlNodeList, expected);
+
+ // Unparse again in Calcite dialect (which we can parse), and
+ // minimal parentheses.
+ final String sql1 = toSqlString(sqlNodeList);
+
+ // Parse and unparse again.
+ SqlNodeList sqlNodeList2;
+ final Quoting q = quoting;
+ try {
+ quoting = Quoting.DOUBLE_QUOTE;
+ sqlNodeList2 = parseStmtsAndHandleEx(sql1);
+ } finally {
+ quoting = q;
+ }
+ final String sql2 = toSqlString(sqlNodeList2);
+
+ // Should be the same as we started with.
+ assertEquals(sql1, sql2);
+
+ // Now unparse again in the null dialect.
+ // If the unparser is not including sufficient parens to override
+ // precedence, the problem will show up here.
+ checkList(sqlNodeList2, expected);
+ }
+
@Override public void check(String sql, String expected) {
SqlNode sqlNode = parseStmtAndHandleEx(sql);
@@ -8822,7 +9019,8 @@ public class SqlParserTest {
assertEquals(expected, linux(actual2));
}
- @Override public void checkFails(String sql, String expectedMsgPattern) {
+ @Override public void checkFails(String sql,
+ boolean list, String expectedMsgPattern) {
// Do nothing. We're not interested in unparsing invalid SQL
}
@@ -8866,7 +9064,7 @@ public class SqlParserTest {
if (expression) {
getTester().checkExpFails(sql, expectedMsgPattern);
} else {
- getTester().checkFails(sql, expectedMsgPattern);
+ getTester().checkFails(sql, false, expectedMsgPattern);
}
return this;
}
@@ -8889,6 +9087,28 @@ public class SqlParserTest {
}
}
+ /** Helper class for building fluent code,
+ * similar to {@link Sql}, but used to manipulate
+ * a list of statements, such as
+ * {@code sqlList("select * from a;").ok();}. */
+ protected class SqlList {
+ private final String sql;
+
+ SqlList(String sql) {
+ this.sql = sql;
+ }
+
+ public SqlList ok(String... expected) {
+ getTester().checkList(sql, ImmutableList.copyOf(expected));
+ return this;
+ }
+
+ public SqlList fails(String expectedMsgPattern) {
+ getTester().checkFails(sql, true, expectedMsgPattern);
+ return this;
+ }
+ }
+
/** Runs tests on period operators such as OVERLAPS, IMMEDIATELY PRECEDES. */
private class Checker {
final String op;
diff --git a/site/_docs/reference.md b/site/_docs/reference.md
index 8ac4581..1317803 100644
--- a/site/_docs/reference.md
+++ b/site/_docs/reference.md
@@ -96,6 +96,9 @@ statement:
| delete
| query
+statementList:
+ statement [ ';' statement ]* [ ';' ]
+
setStatement:
[ ALTER ( SYSTEM | SESSION ) ] SET identifier '=' expression
