Repository: lens Updated Branches: refs/heads/master 39252f9af -> 182f6dcac
LENS-1355 : Fix parsing fields named 'date' Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/182f6dca Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/182f6dca Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/182f6dca Branch: refs/heads/master Commit: 182f6dcacc0226dc0e070edaa3fc8ba4aa3e6146 Parents: 39252f9 Author: Rajat Khandelwal <[email protected]> Authored: Thu Oct 13 12:14:30 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Thu Oct 13 12:14:30 2016 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/AggregateResolver.java | 2 +- .../lens/cube/parse/CubeQueryContext.java | 4 +-- .../apache/lens/cube/parse/GroupbyResolver.java | 4 +-- .../org/apache/lens/cube/parse/HQLParser.java | 26 +++++++++++++++++--- .../apache/lens/cube/parse/TestHQLParser.java | 9 ++++--- 5 files changed, 33 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index c522061..9658100 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -119,7 +119,7 @@ class AggregateResolver implements ContextRewriter { cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator(); itrContext.hasNext();) { for (Iterator<ExprColumn.ExprSpec> itrCol = itrContext.next().getExprCol().getExpressionSpecs().iterator(); itrCol.hasNext();) { - ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr()); + ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr(), cubeql.getConf()); if (HQLParser.isAggregateAST(exprAST)) { return true; } http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 6d53d00..e83ae76 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -886,10 +886,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { String table = entry.getKey(); String rangeWhere = entry.getValue(); if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); + ASTNode rangeAST = HQLParser.parseExpr(rangeWhere, conf); range.getParent().setChild(range.getChildIndex(), rangeAST); } - fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString())); + fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString(), conf)); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index 26ae1e7..216ae52 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -71,7 +71,7 @@ class GroupbyResolver implements ContextRewriter { String expr = sel.getExprWithoutAlias(); if (!groupByExprs.contains(expr)) { if (!sel.isAggregate()) { - ASTNode exprAST = HQLParser.parseExpr(expr); + ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf()); ASTNode groupbyAST = cubeql.getGroupByAST(); if (!isConstantsUsed(exprAST)) { if (groupbyAST != null) { @@ -130,7 +130,7 @@ class GroupbyResolver implements ContextRewriter { int index = 0; for (String expr : groupByExprs) { if (!contains(selectExprs, expr)) { - ASTNode exprAST = HQLParser.parseExpr(expr); + ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf()); addChildAtIndex(index, cubeql.getSelectAST(), exprAST); index++; } http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 1976248..8a70535 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -31,6 +31,7 @@ import java.util.regex.Pattern; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -38,6 +39,8 @@ import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.parse.*; import org.antlr.runtime.CommonToken; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.TokenRewriteStream; import org.antlr.runtime.tree.Tree; import com.google.common.base.Optional; @@ -196,15 +199,32 @@ public final class HQLParser { } public static ASTNode parseExpr(String expr) throws LensException { - ParseDriver driver = new ParseDriver(); + return parseExpr(expr, null); + } + public static ASTNode parseExpr(String expr, Configuration conf) throws LensException { ASTNode tree; try { - tree = driver.parseExpression(expr); - } catch (ParseException e) { + tree = parseExpression(expr, conf); + } catch (ParseException|RecognitionException e) { throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(), e, expr); } return ParseUtils.findRootNonNullToken(tree); } + public static ASTNode parseExpression(String command, Configuration conf) + throws ParseException, RecognitionException { + ParseDriver driver = new ParseDriver(); + ParseDriver.HiveLexerX lexer = driver.new HiveLexerX(driver.new ANTLRNoCaseStringStream(command)); + if (conf != null) { + lexer.setHiveConf(conf); + } + TokenRewriteStream tokens = new TokenRewriteStream(lexer); + HiveParser parser = new HiveParser(tokens); + if (conf != null) { + parser.setHiveConf(conf); + } + parser.setTreeAdaptor(ParseDriver.adaptor); + return (ASTNode)parser.expression().getTree(); + } public static void printAST(ASTNode node) { try { http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java index 2619cb0..3165d3a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java @@ -494,15 +494,16 @@ public class TestHQLParser { @DataProvider public Object[][] exprDataProvider() { return new Object[][] { - {"a.b", true}, - {"a.date", false}, + {"a.b", null, true}, + {"a.date", null, false}, + {"a.date", conf, true}, }; } @Test(dataProvider = "exprDataProvider") - public void testParseExpr(String expr, boolean success) { + public void testParseExpr(String expr, HiveConf conf, boolean success) { try { - HQLParser.parseExpr(expr); + HQLParser.parseExpr(expr, conf); Assert.assertTrue(success); } catch (LensException e) { Assert.assertFalse(success);
