EAGLE-330 Fix Hive ql.Parser Hive ql.Parser cant parser a hive query sql with keywords https://issues.apache.org/jira/browse/EAGLE-330
Author: @wgyumg <wgy...@gmail.com> Reviewer: @pkuwm <h...@ebay.com> Closes: #228 Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/cbf3c76c Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/cbf3c76c Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/cbf3c76c Branch: refs/heads/master Commit: cbf3c76ce148b73379698d9626df6e1baa5162ab Parents: f51474d Author: yonzhang <yonzhang2...@gmail.com> Authored: Tue Jun 7 16:01:41 2016 -0700 Committer: yonzhang <yonzhang2...@gmail.com> Committed: Tue Jun 7 16:01:41 2016 -0700 ---------------------------------------------------------------------- .../apache/eagle/security/hive/ql/Parser.java | 46 ++++++++++++++++++-- .../eagle/security/hive/ql/TestParser.java | 30 ++++++++++++- 2 files changed, 70 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java index 5ab0f4b..19ce003 100644 --- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java +++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java @@ -16,11 +16,19 @@ */ package org.apache.eagle.security.hive.ql; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.Token; +import org.antlr.runtime.TokenRewriteStream; +import org.antlr.runtime.TokenStream; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.CommonTreeAdaptor; +import org.antlr.runtime.tree.TreeAdaptor; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.lib.Node; +import org.apache.hadoop.hive.ql.parse.ASTErrorNode; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +41,21 @@ import java.util.Set; public class Parser { private static final Logger LOG = LoggerFactory.getLogger(Parser.class); + static final TreeAdaptor adaptor = new CommonTreeAdaptor() { + @Override + public Object create(Token payload) { + return new ASTNode(payload); + } + @Override + public Object dupNode(Object t) { + return create(((CommonTree)t).token); + } + @Override + public Object errorNode(TokenStream input, Token start, Token stop, RecognitionException e) { + return new ASTErrorNode(input, start, stop, e); + } + }; + private Set<String> tableSet; private Set<String> columnSet; private Map<String, String> columnAliasMap; @@ -66,11 +89,26 @@ public class Parser { * Parse an Hive QL into an Abstract Syntax Tree(AST). * @param query * @return - * @throws ParseException + * @throws RecognitionException */ - public ASTNode generateAST(String query) throws ParseException { + public ASTNode generateAST(String query) throws RecognitionException { + // https://issues.apache.org/jira/browse/HIVE-10731 + // https://issues.apache.org/jira/browse/HIVE-6617 + HiveConf hiveConf = new HiveConf(); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS, false); + ParseDriver pd = new ParseDriver(); - return pd.parse(query); + ParseDriver.ANTLRNoCaseStringStream antlrStream = pd.new ANTLRNoCaseStringStream(query); + ParseDriver.HiveLexerX lexer = pd.new HiveLexerX(antlrStream); + lexer.setHiveConf(hiveConf); + TokenRewriteStream tokens = new TokenRewriteStream(lexer); + + HiveParser parser = new HiveParser(tokens); + parser.setHiveConf(hiveConf); + parser.setTreeAdaptor(adaptor); + HiveParser.statement_return r = parser.statement(); + + return (ASTNode)r.getTree(); } private void parseQL(ASTNode ast) { http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java index a5f5f82..0bd994f 100644 --- a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java +++ b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.ParseException; +import org.antlr.runtime.RecognitionException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -67,7 +67,7 @@ public class TestParser { } } - public void printQueryAST(String query) throws ParseException { + public void printQueryAST(String query) throws RecognitionException { ASTNode root = parser.generateAST(query); printTree(root, 0); } @@ -209,6 +209,32 @@ public class TestParser { } @Test + public void testQueryWithKeyWords() throws Exception { + String query = "SELECT id, user FROM db.table1"; + String expectedOperation = "SELECT"; + String expectedInsertTable = null; + Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>(); + Set<String> set = new HashSet<String>(); + set.add("id"); + set.add("user"); + expectedTableColumn.put("db.table1", set); + _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn); + } + + @Test + public void testInsertTableWithKeyWords() throws Exception { + String query = "INSERT OVERWRITE TABLE table2 SELECT id, user FROM db.table1"; + String expectedOperation = "SELECT"; + String expectedInsertTable = "table2"; + Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>(); + Set<String> set = new HashSet<String>(); + set.add("id"); + set.add("user"); + expectedTableColumn.put("db.table1", set); + _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn); + } + + @Test public void testCreateTable() throws Exception { String query = "CREATE TABLE page_view(viewTime INT, userid BIGINT,\n" + " page_url STRING, referrer_url STRING,\n" +