Repository: lens Updated Branches: refs/heads/master ff891e2cf -> bf4c0bec0
LENS-851 : Replace columns with aliases in where clause of the inner query Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bf4c0bec Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bf4c0bec Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bf4c0bec Branch: refs/heads/master Commit: bf4c0bec023307417de75f4c13ed1c344fc1f06e Parents: ff891e2 Author: Rajat Khandelwal <pro...@apache.org> Authored: Sat Dec 12 15:30:23 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Sat Dec 12 15:30:23 2015 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/lens-errors.conf | 25 +++++-- .../lens/cube/error/LensCubeErrorCode.java | 6 +- .../apache/lens/cube/metadata/ExprColumn.java | 60 ++++++++++------- .../lens/cube/metadata/MetastoreUtil.java | 16 +++++ .../apache/lens/cube/parse/CandidateFact.java | 8 +-- .../apache/lens/cube/parse/GroupbyResolver.java | 15 +---- .../org/apache/lens/cube/parse/HQLParser.java | 11 +++- .../lens/cube/parse/SingleFactHQLContext.java | 8 +-- .../parse/SingleFactMultiStorageHQLContext.java | 68 +++++++++++++++----- .../apache/lens/cube/parse/UnionHQLContext.java | 2 +- .../cube/metadata/TestCubeMetastoreClient.java | 10 +-- .../lens/cube/metadata/TestExprColumn.java | 20 +++--- .../apache/lens/cube/parse/CubeTestSetup.java | 2 +- .../lens/cube/parse/TestCubeRewriter.java | 4 +- .../apache/lens/server/metastore/JAXBUtils.java | 8 +-- 15 files changed, 161 insertions(+), 102 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index ca8562f..c880543 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -284,9 +284,15 @@ lensCubeErrorsForQuery = [ } { - errorCode = 3031 - httpStatusCode = ${BAD_REQUEST} - errorMsg = "The query is answerable from two storages but union is disabled." + errorCode = 3031 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "The query is answerable from two storages but union is disabled." + } + + { + errorCode = 3032 + httpStatusCode = ${INTERNAL_SERVER_ERROR} + errorMsg = "Could not parse expression %s" } ] @@ -298,10 +304,17 @@ lensCubeErrorsForMetastore = [ } { - errorCode = 3102 - httpStatusCode = ${BAD_REQUEST} - errorMsg = "No timeline found for fact=%s, storage=%s, update period=%s, partition column=%s." + errorCode = 3102 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "No timeline found for fact=%s, storage=%s, update period=%s, partition column=%s." } + + { + errorCode = 3103 + httpStatusCode = ${BAD_REQUEST} + errorMsg = "The Expression %s is Not Parsable." + } + ] lensCubeErrors = ${lensCubeErrorsForQuery}${lensCubeErrorsForMetastore} http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java index 6c5dc2f..68cd80b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java @@ -54,10 +54,12 @@ public enum LensCubeErrorCode { NO_CANDIDATE_FACT_AVAILABLE(3028, 1200), NO_CANDIDATE_DIM_STORAGE_TABLES(3029, 1300), NO_STORAGE_TABLE_AVAIABLE(3030, 1400), - STORAGE_UNION_DISABLED(3031, 100), + STORAGE_UNION_DISABLED(3031, 1500), + COULD_NOT_PARSE_EXPRESSION(3032, 1500), // Error codes greater than 3100 are errors while doing a metastore operation. ERROR_IN_ENTITY_DEFINITION(3101, 100), - TIMELINE_ABSENT(3102, 100); + TIMELINE_ABSENT(3102, 100), + EXPRESSION_NOT_PARSABLE(3103, 1500); public LensErrorInfo getLensErrorInfo() { return this.errorInfo; http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java index b418517..da87e31 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java @@ -23,17 +23,14 @@ import java.io.UnsupportedEncodingException; import java.util.*; import org.apache.lens.cube.parse.HQLParser; +import org.apache.lens.server.api.error.LensException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.ParseException; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import lombok.*; public class ExprColumn extends CubeColumn { public static final char EXPRESSION_DELIMITER = '|'; @@ -46,11 +43,11 @@ public class ExprColumn extends CubeColumn { private int hashCode; // for backward compatibility - public ExprColumn(FieldSchema column, String displayString, String expression) { + public ExprColumn(FieldSchema column, String displayString, String expression) throws LensException { this(column, displayString, new ExprSpec(expression, null, null)); } - public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) { + public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) throws LensException { super(column.getName(), column.getComment(), displayString, null, null, 0.0); if (expressions == null || expressions.length == 0) { @@ -124,6 +121,7 @@ public class ExprColumn extends CubeColumn { public static class ExprSpec { @Getter @Setter + @NonNull private String expr; @Getter @Setter @@ -136,20 +134,18 @@ public class ExprColumn extends CubeColumn { private boolean hasHashCode = false; private transient int hashCode; - public ExprSpec(String expr, Date startTime, Date endTime) { + public ExprSpec(@NonNull String expr, Date startTime, Date endTime) throws LensException { this.expr = expr; this.startTime = startTime; this.endTime = endTime; + // validation + getASTNode(); } - public synchronized ASTNode getASTNode() { + public synchronized ASTNode getASTNode() throws LensException { if (astNode == null) { - try { - if (StringUtils.isNotBlank(expr)) { - astNode = HQLParser.parseExpr(getExpr()); - } - } catch (ParseException e) { - throw new IllegalArgumentException("Expression can't be parsed: " + getExpr(), e); + if (StringUtils.isNotBlank(expr)) { + astNode = MetastoreUtil.parseExpr(getExpr()); } } return astNode; @@ -160,8 +156,14 @@ public class ExprColumn extends CubeColumn { if (!hasHashCode) { final int prime = 31; int result = 1; - if (getASTNode() != null) { - String exprNormalized = HQLParser.getString(getASTNode()); + ASTNode astNode; + try { + astNode = getASTNode(); + } catch (LensException e) { + throw new IllegalArgumentException(e); + } + if (astNode != null) { + String exprNormalized = HQLParser.getString(astNode); result = prime * result + exprNormalized.hashCode(); } result = prime * result + ((getStartTime() == null) ? 0 : COLUMN_TIME_FORMAT.get().format( @@ -262,9 +264,17 @@ public class ExprColumn extends CubeColumn { return false; } // Compare expressions for both - compare ASTs - List<ASTNode> myExpressions = getExpressionASTList(); - List<ASTNode> otherExpressions = other.getExpressionASTList(); - + List<ASTNode> myExpressions, otherExpressions; + try { + myExpressions = getExpressionASTList(); + } catch (LensException e) { + throw new IllegalArgumentException(e); + } + try { + otherExpressions = other.getExpressionASTList(); + } catch (LensException e) { + throw new IllegalArgumentException(e); + } for (int i = 0; i < myExpressions.size(); i++) { if (!HQLParser.equalsAST(myExpressions.get(i), otherExpressions.get(i))) { return false; @@ -316,11 +326,11 @@ public class ExprColumn extends CubeColumn { * * @return the ast */ - public ASTNode getAst() { + public ASTNode getAst() throws LensException { return getExpressionASTList().get(0); } - public List<ASTNode> getExpressionASTList() { + public List<ASTNode> getExpressionASTList() throws LensException { synchronized (expressionSet) { if (astNodeList.isEmpty()) { for (ExprSpec expr : expressionSet) { @@ -366,15 +376,15 @@ public class ExprColumn extends CubeColumn { * Add an expression to existing set of expressions for this column * * @param expression - * @throws ParseException + * @throws LensException */ - public void addExpression(ExprSpec expression) throws ParseException { + public void addExpression(ExprSpec expression) throws LensException { if (expression == null || expression.getExpr().isEmpty()) { throw new IllegalArgumentException("Empty expression not allowed"); } // Validate if expression can be correctly parsed - HQLParser.parseExpr(expression.getExpr()); + MetastoreUtil.parseExpr(expression.getExpr()); synchronized (expressionSet) { expressionSet.add(expression); } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 4ec049c..deb5368 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -19,14 +19,20 @@ package org.apache.lens.cube.metadata; +import static org.apache.lens.cube.error.LensCubeErrorCode.EXPRESSION_NOT_PARSABLE; import static org.apache.lens.cube.metadata.MetastoreConstants.*; import java.text.ParseException; import java.util.*; +import org.apache.lens.server.api.error.LensException; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.ParseUtils; import com.google.common.collect.Sets; @@ -536,4 +542,14 @@ public class MetastoreUtil { } return null; } + public static ASTNode parseExpr(String expr) throws LensException { + ParseDriver driver = new ParseDriver(); + ASTNode tree; + try { + tree = driver.parseExpression(expr); + } catch (org.apache.hadoop.hive.ql.parse.ParseException e) { + throw new LensException(EXPRESSION_NOT_PARSABLE.getLensErrorInfo(), e, e.getMessage(), expr); + } + return ParseUtils.findRootNonNullToken(tree); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 1884bde..2338ba7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.apache.hadoop.hive.ql.session.SessionState; import org.antlr.runtime.CommonToken; @@ -159,12 +158,7 @@ public class CandidateFact implements CandidateTable { TimeRange range = cubeql.getTimeRanges().get(i); String rangeWhere = rangeToWhereClause.get(range); if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST; - try { - rangeAST = HQLParser.parseExpr(rangeWhere); - } catch (ParseException e) { - throw new LensException(e); - } + ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); rangeAST.setParent(timenodes.get(i).parent); timenodes.get(i).parent.setChild(timenodes.get(i).childIndex, rangeAST); } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index 97088a1..da74713 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -31,7 +31,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; -import org.apache.hadoop.hive.ql.parse.ParseException; import org.antlr.runtime.CommonToken; import org.antlr.runtime.tree.Tree; @@ -73,12 +72,7 @@ class GroupbyResolver implements ContextRewriter { if (!groupByExprs.contains(expr)) { if (!cubeql.isAggregateExpr(expr)) { - ASTNode exprAST; - try { - exprAST = HQLParser.parseExpr(expr); - } catch (ParseException e) { - throw new LensException(e); - } + ASTNode exprAST = HQLParser.parseExpr(expr); ASTNode groupbyAST = cubeql.getGroupByAST(); if (!isConstantsUsed(exprAST)) { if (groupbyAST != null) { @@ -140,12 +134,7 @@ class GroupbyResolver implements ContextRewriter { int index = 0; for (String expr : groupByExprs) { if (!contains(cubeql, selectExprs, expr)) { - ASTNode exprAST; - try { - exprAST = HQLParser.parseExpr(expr); - } catch (ParseException e) { - throw new LensException(e); - } + ASTNode exprAST = HQLParser.parseExpr(expr); addChildAtIndex(index, cubeql.getSelectAST(), exprAST); index++; } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 9a9d134..7cea7d5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -18,10 +18,10 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.error.LensCubeErrorCode.COULD_NOT_PARSE_EXPRESSION; import static org.apache.lens.cube.error.LensCubeErrorCode.SYNTAX_ERROR; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; -import static org.apache.hadoop.hive.ql.parse.HiveParser.Number; import java.io.IOException; import java.lang.reflect.Field; @@ -170,9 +170,14 @@ public final class HQLParser { return tree; } - public static ASTNode parseExpr(String expr) throws ParseException { + public static ASTNode parseExpr(String expr) throws LensException { ParseDriver driver = new ParseDriver(); - ASTNode tree = driver.parseExpression(expr); + ASTNode tree; + try { + tree = driver.parseExpression(expr); + } catch (ParseException e) { + throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(), e, e.getMessage()); + } return ParseUtils.findRootNonNullToken(tree); } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java index f7271e5..de52b0a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java @@ -26,7 +26,6 @@ import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.ParseException; /** * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being @@ -67,12 +66,7 @@ class SingleFactHQLContext extends DimOnlyHQLContext { String rangeWhere = entry.getKey(); if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST; - try { - rangeAST = HQLParser.parseExpr(rangeWhere); - } catch (ParseException e) { - throw new LensException(e); - } + ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); rangeAST.setParent(range.getParent()); range.getParent().setChild(range.getChildIndex(), rangeAST); } http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 418ef5a..96b1d05 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -19,7 +19,7 @@ package org.apache.lens.cube.parse; -import static org.apache.lens.cube.parse.HQLParser.getString; +import static org.apache.lens.cube.parse.HQLParser.*; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; @@ -66,7 +66,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { @Override public int hashCode() { if (!hashCodeComputed) { - hashCode = HQLParser.getString(ast).hashCode(); + hashCode = getString(ast).hashCode(); hashCodeComputed = true; } return hashCode; @@ -74,8 +74,8 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { @Override public boolean equals(Object o) { - return o instanceof HashableASTNode && this.hashCode() == o.hashCode() && HQLParser.getString(this.getAST()) - .trim().equalsIgnoreCase(HQLParser.getString(((HashableASTNode) o).getAST()).trim()); + return o instanceof HashableASTNode && this.hashCode() == o.hashCode() && getString(this.getAST()) + .trim().equalsIgnoreCase(getString(((HashableASTNode) o).getAST()).trim()); } } @@ -86,6 +86,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { super(query, fact); processSelectAST(); processGroupByAST(); + processWhereAST(); processHavingAST(); processOrderByAST(); processLimit(); @@ -94,10 +95,10 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { private void processSelectAST() { query.getSelectFinalAliases().clear(); - ASTNode originalSelectAST = HQLParser.copyAST(query.getSelectAST()); + ASTNode originalSelectAST = copyAST(query.getSelectAST()); query.setSelectAST(new ASTNode(originalSelectAST.getToken())); ASTNode outerSelectAST = processExpression(originalSelectAST); - setSelect(HQLParser.getString(outerSelectAST)); + setSelect(getString(outerSelectAST)); } private void processGroupByAST() { @@ -106,16 +107,25 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } } + private void processWhereAST() throws LensException { + for (String storageTable : fact.getStorgeWhereClauseMap().keySet()) { + ASTNode tree = parseExpr(fact.getStorgeWhereClauseMap().get(storageTable)); + ASTNode replaced = replaceAST(tree); + fact.getStorgeWhereClauseMap().put(storageTable, getString(replaced)); + } + } + private void processHavingAST() throws LensException { if (query.getHavingAST() != null) { - setHaving(HQLParser.getString(processExpression(query.getHavingAST()))); + setHaving(getString(processExpression(query.getHavingAST()))); query.setHavingAST(null); } } + private void processOrderByAST() { if (query.getOrderByAST() != null) { - setOrderby(HQLParser.getString(processExpression(query.getOrderByAST()))); + setOrderby(getString(processExpression(query.getOrderByAST()))); query.setOrderByAST(null); } } @@ -124,6 +134,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { setLimit(query.getLimitValue()); query.setLimitValue(null); } + /* Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the inner query - outer query dynamics. The resultant AST is supposed to be used in outer query. @@ -147,11 +158,11 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { if (astNode == null) { return null; } - if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { - return innerToOuterASTs.get(new HashableASTNode(astNode)); - } - if (HQLParser.isAggregateAST(astNode)) { - ASTNode innerSelectASTWithoutAlias = HQLParser.copyAST(astNode); + if (isAggregateAST(astNode)) { + if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { + return innerToOuterASTs.get(new HashableASTNode(astNode)); + } + ASTNode innerSelectASTWithoutAlias = copyAST(astNode); ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); innerSelectExprAST.addChild(innerSelectASTWithoutAlias); String alias = decideAlias(astNode); @@ -164,8 +175,11 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { outerAST.addChild(dotAST); innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); return outerAST; - } else if (HQLParser.isTableColumnAST(astNode)) { - ASTNode innerSelectASTWithoutAlias = HQLParser.copyAST(astNode); + } else if (isTableColumnAST(astNode)) { + if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { + return innerToOuterASTs.get(new HashableASTNode(astNode)); + } + ASTNode innerSelectASTWithoutAlias = copyAST(astNode); ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); innerSelectExprAST.addChild(innerSelectASTWithoutAlias); String alias = decideAlias(astNode); @@ -186,6 +200,30 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } } + /** + * Transforms the inner query's AST so that aliases are used now instead of column names. + * Does so in-place, without creating new ASTNode instances. + * @param astNode inner query's AST Node to transform + * @return Transformed AST Node. + */ + private ASTNode replaceAST(ASTNode astNode) { + if (astNode == null) { + return null; + } + if (isAggregateAST(astNode) || isTableColumnAST(astNode)) { + if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { + ASTNode ret = innerToOuterASTs.get(new HashableASTNode(astNode)); + // Set parent null for quicker GC + astNode.setParent(null); + return ret; + } + } + for (int i = 0; i < astNode.getChildCount(); i++) { + astNode.setChild(i, replaceAST((ASTNode) astNode.getChild(i))); + } + return astNode; + } + private void addToInnerSelectAST(ASTNode selectExprAST) { if (query.getSelectAST() == null) { query.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT))); http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java index c9ba561..e6ee989 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java @@ -33,7 +33,7 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public abstract class UnionHQLContext extends SimpleHQLContext { protected final CubeQueryContext query; - private final CandidateFact fact; + protected final CandidateFact fact; List<HQLContextInterface> hqlContexts = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index c6ce6ad..0fef13f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -43,11 +43,7 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.ParseException; +import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.mapred.SequenceFileInputFormat; @@ -305,7 +301,7 @@ public class TestCubeMetastoreClient { new DerivedCube(derivedCubeNameWithProps, measures, dimensions, CUBE_PROPERTIES, 0L, cubeWithProps); } - private static void defineUberDims() { + private static void defineUberDims() throws LensException { // Define zip dimension zipAttrs.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "code"))); zipAttrs.add(new BaseDimAttribute(new FieldSchema("f1", "string", "field1"))); @@ -411,7 +407,7 @@ public class TestCubeMetastoreClient { expr1.setExpr("contact(countrydim.name"); stateCountryExpr.addExpression(expr1); fail("Expected add expression to fail because of syntax error"); - } catch (ParseException exc) { + } catch (LensException exc) { // Pass } city.alterExpression(stateCountryExpr); http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java index 8770f1a..0153b2d 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java @@ -28,7 +28,9 @@ import java.util.Iterator; import java.util.Map; import java.util.TimeZone; +import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; +import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -142,7 +144,7 @@ public class TestExprColumn { } @Test - public void testExprColumnCreationErrors() { + public void testExprColumnCreationErrors() throws LensException { FieldSchema colSchema = new FieldSchema("errorColumn", "double", "multi exprcol"); // no expression spec passed @@ -157,16 +159,16 @@ public class TestExprColumn { try { ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec(null, null, null)); fail(col1 + " should not be created"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:0")); + } catch (NullPointerException e) { + // pass } // Parse error in expr passed in exprspec try { ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("(a+b", null, null)); fail(col1 + " should not be created"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b"), e.getMessage()); + } catch (LensException e) { + assertEquals(e.getErrorCode(), LensCubeErrorCode.EXPRESSION_NOT_PARSABLE.getLensErrorInfo().getErrorCode()); } // Parse error in expr passed in exprspec @@ -174,8 +176,8 @@ public class TestExprColumn { ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("a + b", null, null), new ExprSpec("(a+b", null, null)); fail(col1 + " should not be created"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b")); + } catch (LensException e) { + assertEquals(e.getErrorCode(), LensCubeErrorCode.EXPRESSION_NOT_PARSABLE.getLensErrorInfo().getErrorCode()); } // no expression passed in exprspec @@ -183,8 +185,8 @@ public class TestExprColumn { ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpecAt1", new ExprSpec("a + b", null, null), new ExprSpec(null, null, null)); fail(col1 + " should not be created"); - } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:1")); + } catch (NullPointerException e) { + // pass } // startTime after endTime http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 2a50d74..3f01dbe 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -1561,7 +1561,7 @@ public class CubeTestSetup { } // DimWithTwoStorages - private void createCityTable(CubeMetastoreClient client) throws HiveException, ParseException { + private void createCityTable(CubeMetastoreClient client) throws HiveException, ParseException, LensException { Set<CubeDimAttribute> cityAttrs = new HashSet<CubeDimAttribute>(); cityAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code"))); cityAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "city name"))); http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 3be9406..0f05556 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -416,7 +416,7 @@ public class TestCubeRewriter extends TestQueryRewrite { conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); hqlQuery = rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` " - + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); + + "from testCube where cityid = 'a' and zipcode = 'b' and " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider, "SELECT testcube.alias0 as `City ID`, sum(testcube.alias1) + max(testcube.alias2), " @@ -426,7 +426,7 @@ public class TestCubeRewriter extends TestQueryRewrite { "select testcube.cityid as `alias0`, sum(testcube.msr2) as `alias1`, " + "max(testcube.msr3) as `alias2`, " + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`", - null, "group by testcube.cityid"); + "testcube.alias0 = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid"); compareQueries(hqlQuery, expected); http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java index a5883f7..817c84c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java @@ -316,7 +316,7 @@ public final class JAXBUtils { return xes; } - private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) { + private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) throws LensException { List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size()); for (XExprSpec xes : xesList) { esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime()))); @@ -478,7 +478,7 @@ public final class JAXBUtils { return jc; } - public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) { + public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws LensException { ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(), xe.getDescription()), xe.getDisplayString(), @@ -598,7 +598,7 @@ public final class JAXBUtils { return null; } - Storage storage = null; + Storage storage; try { Class<?> clazz = Class.forName(xs.getClassname()); Constructor<?> constructor = clazz.getConstructor(String.class); @@ -924,7 +924,7 @@ public final class JAXBUtils { return ret; } - public static Dimension dimensionFromXDimension(XDimension dimension) { + public static Dimension dimensionFromXDimension(XDimension dimension) throws LensException { Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>(); for (XDimAttribute xd : dimension.getAttributes().getDimAttribute()) { dims.add(hiveDimAttrFromXDimAttr(xd));