Repository: lens Updated Branches: refs/heads/master 46e915367 -> 382881940
LENS-614 : Support single columns as expressions - more like alias Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/38288194 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/38288194 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/38288194 Branch: refs/heads/master Commit: 3828819403c7c27c25e1d7d2421d0dcc5f506cdf Parents: 46e9153 Author: Amareshwari Sriramadasu <[email protected]> Authored: Thu May 12 10:06:16 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Thu May 12 10:06:16 2016 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/metadata/ExprColumn.java | 26 ++-- .../lens/cube/metadata/MetastoreUtil.java | 16 ++- .../lens/cube/parse/AggregateResolver.java | 15 +- .../apache/lens/cube/parse/AliasReplacer.java | 15 +- .../apache/lens/cube/parse/CandidateFact.java | 8 +- .../apache/lens/cube/parse/ColumnResolver.java | 4 +- .../lens/cube/parse/ExpressionResolver.java | 31 ++--- .../org/apache/lens/cube/parse/HQLParser.java | 13 -- .../parse/SingleFactMultiStorageHQLContext.java | 7 +- .../cube/metadata/TestCubeMetastoreClient.java | 36 ++--- .../apache/lens/cube/parse/CubeTestSetup.java | 12 ++ .../lens/cube/parse/TestExpressionResolver.java | 136 +++++++++++++++++++ 12 files changed, 233 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java index 1c9d6d2..52feeb8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java @@ -36,8 +36,8 @@ public class ExprColumn extends CubeColumn { public static final char EXPRESSION_DELIMITER = '|'; public static final char EXPRESSION_SPEC_DELIMITER = ':'; private static final String EXPRESSION_ENCODED = "true"; - private final Set<ExprSpec> expressionSet = new LinkedHashSet<ExprSpec>(); - private List<ASTNode> astNodeList = new ArrayList<ASTNode>(); + private final Set<ExprSpec> expressionSet = new LinkedHashSet<>(); + private List<ASTNode> astNodeList = new ArrayList<>(); private final String type; private boolean hasHashCode = false; private int hashCode; @@ -98,17 +98,17 @@ public class ExprColumn extends CubeColumn { String decodedExpr = isExpressionBase64Encoded ? new String(Base64.decodeBase64(exprSpecStrs[0]), "UTF-8") : exprSpecStrs[0]; ExprSpec exprSpec = new ExprSpec(); - exprSpec.setExpr(decodedExpr); + exprSpec.expr = decodedExpr; if (exprSpecStrs.length > 1) { // start time and end time serialized if (StringUtils.isNotBlank(exprSpecStrs[1])) { // start time available - exprSpec.setStartTime(getDate(exprSpecStrs[1])); + exprSpec.startTime = getDate(exprSpecStrs[1]); } if (exprSpecStrs.length > 2) { if (StringUtils.isNotBlank(exprSpecStrs[2])) { // end time available - exprSpec.setEndTime(getDate(exprSpecStrs[2])); + exprSpec.endTime = getDate(exprSpecStrs[2]); } } } @@ -126,14 +126,11 @@ public class ExprColumn extends CubeColumn { @ToString(exclude = {"astNode", "hasHashCode", "hashCode"}) public static class ExprSpec { @Getter - @Setter @NonNull private String expr; @Getter - @Setter private Date startTime; @Getter - @Setter private Date endTime; private transient ASTNode astNode; @@ -145,18 +142,25 @@ public class ExprColumn extends CubeColumn { this.startTime = startTime; this.endTime = endTime; // validation - getASTNode(); + initASTNode(); } - public synchronized ASTNode getASTNode() throws LensException { + private synchronized void initASTNode() throws LensException { if (astNode == null) { if (StringUtils.isNotBlank(expr)) { astNode = MetastoreUtil.parseExpr(getExpr()); } } + } + + private ASTNode getASTNode() throws LensException { + initASTNode(); return astNode; } + public ASTNode copyASTNode() throws LensException { + return MetastoreUtil.copyAST(getASTNode()); + } @Override public int hashCode() { if (!hasHashCode) { @@ -340,7 +344,7 @@ public class ExprColumn extends CubeColumn { synchronized (expressionSet) { if (astNodeList.isEmpty()) { for (ExprSpec expr : expressionSet) { - astNodeList.add(expr.getASTNode()); + astNodeList.add(expr.copyASTNode()); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 006c4cb..53cf8af 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -566,7 +566,8 @@ public class MetastoreUtil { } return null; } - public static ASTNode parseExpr(String expr) throws LensException { + + static ASTNode parseExpr(String expr) throws LensException { ParseDriver driver = new ParseDriver(); ASTNode tree; try { @@ -576,4 +577,17 @@ public class MetastoreUtil { } return ParseUtils.findRootNonNullToken(tree); } + + public static ASTNode copyAST(ASTNode original) { + + ASTNode copy = new ASTNode(original); // Leverage constructor + + if (original.getChildren() != null) { + for (Object o : original.getChildren()) { + ASTNode childCopy = copyAST((ASTNode) o); + copy.addChild(childCopy); + } + } + return copy; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 1593a54..ef2ca4e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -27,6 +27,7 @@ import java.util.Iterator; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.CubeMeasure; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; +import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -123,9 +124,9 @@ class AggregateResolver implements ContextRewriter { return HQLParser.getString(clause); } - private void transform(CubeQueryContext cubeql, ASTNode parent, ASTNode node, int nodePos) throws LensException { + private ASTNode transform(CubeQueryContext cubeql, ASTNode parent, ASTNode node, int nodePos) throws LensException { if (node == null) { - return; + return node; } int nodeType = node.getToken().getType(); @@ -145,6 +146,8 @@ class AggregateResolver implements ContextRewriter { expr = HQLParser.getString(wrapped); } cubeql.addAggregateExpr(expr.trim()); + } else { + return wrapped; } } } else { @@ -154,6 +157,7 @@ class AggregateResolver implements ContextRewriter { } } } + return node; } // Wrap an aggregate function around the node if its a measure, leave it @@ -179,8 +183,9 @@ class AggregateResolver implements ContextRewriter { if (cubeql.isCubeMeasure(msrname)) { if (cubeql.getQueriedExprs().contains(colname)) { String alias = cubeql.getAliasForTableName(cubeql.getCube().getName()); - for (ASTNode exprNode : cubeql.getExprCtx().getExpressionContext(colname, alias).getAllASTNodes()) { - transform(cubeql, null, exprNode, 0); + for (ExprSpecContext esc : cubeql.getExprCtx().getExpressionContext(colname, alias).getAllExprs()) { + ASTNode transformedNode = transform(cubeql, null, esc.getFinalAST(), 0); + esc.setFinalAST(transformedNode); } return node; } else { @@ -190,7 +195,7 @@ class AggregateResolver implements ContextRewriter { if (StringUtils.isBlank(aggregateFn)) { throw new LensException(LensCubeErrorCode.NO_DEFAULT_AGGREGATE.getLensErrorInfo(), colname); } - ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION)); + ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION, "TOK_FUNCTION")); ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn)); fnroot.addChild(fnIdentNode); fnroot.addChild(node); http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java index e629731..5b48ca4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java @@ -164,9 +164,9 @@ class AliasReplacer implements ContextRewriter { } } - static void replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) { + static ASTNode replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) { if (node == null) { - return; + return node; } int nodeType = node.getToken().getType(); @@ -175,7 +175,7 @@ class AliasReplacer implements ContextRewriter { String newAlias = colToTableAlias.get(colName.toLowerCase()); if (StringUtils.isBlank(newAlias)) { - return; + return node; } if (nodeType == HiveParser.DOT) { @@ -194,12 +194,14 @@ class AliasReplacer implements ContextRewriter { dot.addChild(tabRefNode); ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName)); - dot.addChild(colIdentNode); ASTNode parent = (ASTNode) node.getParent(); - - parent.setChild(nodePos, dot); + if (parent != null) { + parent.setChild(nodePos, dot); + } else { + return dot; + } } } else { // recurse down @@ -208,6 +210,7 @@ class AliasReplacer implements ContextRewriter { replaceAliases(child, i, colToTableAlias); } } + return node; } static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) { http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 3f724b6..cd7a02c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -150,13 +150,13 @@ public class CandidateFact implements CandidateTable, QueryAST { // copy ASTs from CubeQueryContext public void copyASTs(CubeQueryContext cubeql) throws LensException { - setSelectAST(HQLParser.copyAST(cubeql.getSelectAST())); - setWhereAST(HQLParser.copyAST(cubeql.getWhereAST())); + setSelectAST(MetastoreUtil.copyAST(cubeql.getSelectAST())); + setWhereAST(MetastoreUtil.copyAST(cubeql.getWhereAST())); if (cubeql.getJoinAST() != null) { - setJoinAST(HQLParser.copyAST(cubeql.getJoinAST())); + setJoinAST(MetastoreUtil.copyAST(cubeql.getJoinAST())); } if (cubeql.getGroupByAST() != null) { - setGroupByAST(HQLParser.copyAST(cubeql.getGroupByAST())); + setGroupByAST(MetastoreUtil.copyAST(cubeql.getGroupByAST())); } } http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java index 58a1d4d..2db5dd1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java @@ -95,7 +95,7 @@ class ColumnResolver implements ContextRewriter { parent = visited.getParent().getNode(); } - if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) { + if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent == null || parent.getToken().getType() != DOT)) { // Take child ident.totext ASTNode ident = (ASTNode) node.getChild(0); String column = ident.getText().toLowerCase(); @@ -123,7 +123,7 @@ class ColumnResolver implements ContextRewriter { // find columns in where tree // if where expression is timerange function, then time range columns are // added - // only if timerange clause shouldn't be replaced with its correspodning + // only if timerange clause shouldn't be replaced with its corresponding // partition column private void getColsForWhereTree(final CubeQueryContext cubeql) throws LensException { if (cubeql.getWhereAST() == null) { http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index c3d15d0..898c438 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -36,10 +36,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser; import org.antlr.runtime.CommonToken; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NonNull; -import lombok.ToString; +import lombok.*; import lombok.extern.slf4j.Slf4j; /** @@ -136,8 +133,8 @@ class ExpressionResolver implements ContextRewriter { for (String col : exprCols) { Set<ExprSpecContext> replacedExpressions = new LinkedHashSet<ExprSpecContext>(); for (ExprSpec es : baseTable.getExpressionByName(col).getExpressionSpecs()) { - ASTNode finalAST = HQLParser.copyAST(baseEsc.getFinalAST()); - replaceColumnInAST(finalAST, col, es.getASTNode()); + ASTNode finalAST = MetastoreUtil.copyAST(baseEsc.getFinalAST()); + replaceColumnInAST(finalAST, col, es.copyASTNode()); ExprSpecContext replacedESC = new ExprSpecContext(baseEsc, es, finalAST, cubeql); nestedExpressions.add(replacedESC); replacedExpressions.add(replacedESC); @@ -201,19 +198,19 @@ class ExpressionResolver implements ContextRewriter { } static class ExprSpecContext implements TrackQueriedColumns { + private Set<ExprSpec> exprSpecs = new LinkedHashSet<>(); @Getter - private Set<ExprSpec> exprSpecs = new LinkedHashSet<ExprSpec>(); - @Getter + @Setter private ASTNode finalAST; @Getter - private Set<Dimension> exprDims = new HashSet<Dimension>(); + private Set<Dimension> exprDims = new HashSet<>(); // for each expression store alias to columns queried @Getter - private Map<String, Set<String>> tblAliasToColumns = new HashMap<String, Set<String>>(); + private Map<String, Set<String>> tblAliasToColumns = new HashMap<>(); ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException { // replaces table names in expression with aliases in the query - finalAST = replaceAlias(exprSpec.getASTNode(), cubeql); + finalAST = replaceAlias(exprSpec.copyASTNode(), cubeql); exprSpecs.add(exprSpec); } public ExprSpecContext(ExprSpecContext nested, ExprSpec current, ASTNode node, @@ -225,7 +222,7 @@ class ExpressionResolver implements ContextRewriter { public void replaceAliasInAST(CubeQueryContext cubeql) throws LensException { AliasReplacer.extractTabAliasForCol(cubeql, this); - AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); + finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); } public void addColumnsQueried(String alias, String column) { Set<String> cols = tblAliasToColumns.get(alias.toLowerCase()); @@ -242,7 +239,7 @@ class ExpressionResolver implements ContextRewriter { } Date getStartTime() { - Set<Date> startTimes = new HashSet<Date>(); + Set<Date> startTimes = new HashSet<>(); for (ExprSpec es : exprSpecs) { if (es.getStartTime() != null) { startTimes.add(es.getStartTime()); @@ -255,7 +252,7 @@ class ExpressionResolver implements ContextRewriter { } Date getEndTime() { - Set<Date> endTimes = new HashSet<Date>(); + Set<Date> endTimes = new HashSet<>(); for (ExprSpec es : exprSpecs) { if (es.getEndTime() != null) { endTimes.add(es.getEndTime()); @@ -721,7 +718,7 @@ class ExpressionResolver implements ContextRewriter { } private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws LensException { - ASTNode finalAST = HQLParser.copyAST(expr); + ASTNode finalAST = MetastoreUtil.copyAST(expr); HQLParser.bft(finalAST, new ASTNodeVisitor() { @Override public void visit(TreeNode visited) { @@ -764,7 +761,7 @@ class ExpressionResolver implements ContextRewriter { ASTNode ident = (ASTNode) current.getChild(0); String column = ident.getText().toLowerCase(); if (toReplace.equals(column)) { - node.setChild(i, HQLParser.copyAST(columnAST)); + node.setChild(i, MetastoreUtil.copyAST(columnAST)); } } else if (current.getToken().getType() == DOT) { // This is for the case where column name is prefixed by table name @@ -777,7 +774,7 @@ class ExpressionResolver implements ContextRewriter { String column = colIdent.getText().toLowerCase(); if (toReplace.equals(column)) { - node.setChild(i, HQLParser.copyAST(columnAST)); + node.setChild(i, MetastoreUtil.copyAST(columnAST)); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 6c2a168..68cdcef 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -310,19 +310,6 @@ public final class HQLParser { return null; } - public static ASTNode copyAST(ASTNode original) { - - ASTNode copy = new ASTNode(original); // Leverage constructor - - if (original.getChildren() != null) { - for (Object o : original.getChildren()) { - ASTNode childCopy = copyAST((ASTNode) o); - copy.addChild(childCopy); - } - } - return copy; - } - /** * Breadth first traversal of AST * http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 9ee94d3..1af031a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -31,6 +31,7 @@ import java.util.Map; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.MetastoreUtil; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.ql.lib.Node; @@ -63,7 +64,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } private void processSelectAST() { - ASTNode originalSelectAST = copyAST(ast.getSelectAST()); + ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST()); ast.setSelectAST(new ASTNode(originalSelectAST.getToken())); ASTNode outerSelectAST = processExpression(originalSelectAST); setSelect(getString(outerSelectAST)); @@ -122,7 +123,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { return innerToOuterASTs.get(new HashableASTNode(astNode)); } - ASTNode innerSelectASTWithoutAlias = copyAST(astNode); + ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); innerSelectExprAST.addChild(innerSelectASTWithoutAlias); String alias = aliasDecider.decideAlias(astNode); @@ -140,7 +141,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { return innerToOuterASTs.get(new HashableASTNode(astNode)); } - ASTNode innerSelectASTWithoutAlias = copyAST(astNode); + ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); innerSelectExprAST.addChild(innerSelectASTWithoutAlias); String alias = aliasDecider.decideAlias(astNode); http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index 1ed6258..d992967 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -198,28 +198,20 @@ public class TestCubeMetastoreClient { "dummy_dim" + i, null, null, null, null, regions)); } - ExprSpec expr1 = new ExprSpec(); - expr1.setExpr("avg(msr1 + msr2)"); - ExprSpec expr2 = new ExprSpec(); - expr2.setExpr("avg(msr2 + msr1)"); - ExprSpec expr3 = new ExprSpec(); - expr3.setExpr("avg(msr1 + msr2 - msr1 + msr1)"); + ExprSpec expr1 = new ExprSpec("avg(msr1 + msr2)", null, null); + ExprSpec expr2 = new ExprSpec("avg(msr2 + msr1)", null, null); + ExprSpec expr3 = new ExprSpec("avg(msr1 + msr2 - msr1 + msr1)", null, null); cubeExpressions.add(new ExprColumn(new FieldSchema("msr5", "double", "fifth measure"), "Avg msr5", expr1, expr2, expr3)); - expr1 = new ExprSpec(); - expr1.setExpr("avg(msr1 + msr2)"); + expr1 = new ExprSpec("avg(msr1 + msr2)", null, null); cubeExpressions.add(new ExprColumn(new FieldSchema("msr5start", "double", "expr measure with start and end times"), "AVG of SUM", expr1)); - expr1 = new ExprSpec(); - expr1.setExpr("dim1 != 'x' AND dim2 != 10 "); - expr2 = new ExprSpec(); - expr2.setExpr("dim1 | dim2 AND dim2 = 'XYZ'"); + expr1 = new ExprSpec("dim1 != 'x' AND dim2 != 10 ", null, null); + expr2 = new ExprSpec("dim1 | dim2 AND dim2 = 'XYZ'", null, null); cubeExpressions.add(new ExprColumn(new FieldSchema("booleancut", "boolean", "a boolean expression"), "Boolean Cut", expr1, expr2)); - expr1 = new ExprSpec(); - expr1.setExpr("substr(dim1, 3)"); - expr2 = new ExprSpec(); - expr2.setExpr("substr(dim2, 3)"); + expr1 = new ExprSpec("substr(dim1, 3)", null, null); + expr2 = new ExprSpec("substr(dim2, 3)", null, null); cubeExpressions.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a subt string expression"), "SUBSTR EXPR", expr1, expr2)); @@ -538,14 +530,12 @@ public class TestCubeMetastoreClient { ExprColumn stateCountryExpr = new ExprColumn(new FieldSchema("stateAndCountry", "String", "state and country together with hiphen as separator"), "State and Country", "concat(citystate.name, \"-\", citycountry.name)"); - ExprSpec expr1 = new ExprSpec(); - expr1.setExpr("concat(countrydim.name, \"-\", countrydim.name)"); + ExprSpec expr1 = new ExprSpec("concat(countrydim.name, \"-\", countrydim.name)", null, null); stateCountryExpr.addExpression(expr1); // Assert expression validation try { - expr1 = new ExprSpec(); - expr1.setExpr("contact(countrydim.name"); + expr1 = new ExprSpec("contact(countrydim.name", null , null); stateCountryExpr.addExpression(expr1); fail("Expected add expression to fail because of syntax error"); } catch (LensException exc) { @@ -867,10 +857,8 @@ public class TestCubeMetastoreClient { cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1", "id", "ref dim"), "dim with tag", null, null, null, null, null, tag1)); - ExprSpec expr1 = new ExprSpec(); - expr1.setExpr("avg(msr1 + msr2)"); - ExprSpec expr2 = new ExprSpec(); - expr2.setExpr("avg(msr2 + msr1)"); + ExprSpec expr1 = new ExprSpec("avg(msr1 + msr2)", null, null); + ExprSpec expr2 = new ExprSpec("avg(msr2 + msr1)", null, null); Set<ExprColumn> cubeExpressions = new HashSet<>(); cubeExpressions.add(new ExprColumn(new FieldSchema("expr_measure", "double", "expression measure"), http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 73e6221..c0035a7 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -584,6 +584,18 @@ public class CubeTestSetup { exprs = new HashSet<ExprColumn>(); exprs.add(new ExprColumn(new FieldSchema("avgmsr", "double", "avg measure"), "Avg Msr", "avg(msr1 + msr2)")); + exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2expr", "double", "measure2"), "Msr2", "msr2)")); + exprs.add(new ExprColumn(new FieldSchema("singlecolmsr2qualifiedexpr", "double", "testcube.measure2"), + "Msr2", "testcube.msr2")); + exprs.add(new ExprColumn(new FieldSchema("singlecoldim1expr", "string", "dim1"), "dim1", "dim1)")); + exprs.add(new ExprColumn(new FieldSchema("singlecoldim1qualifiedexpr", "string", "testcube.dim1"), + "dim1", "testcube.dim1")); + exprs.add(new ExprColumn(new FieldSchema("singlecolchainid", "string", "dim3chain.id"), + "dim3chainid", "dim3chain.id)")); + exprs.add(new ExprColumn(new FieldSchema("singlecolchainrefexpr", "string", "testcube.testDim3id"), + "dim3chainid", "testcube.testDim3id")); + exprs.add(new ExprColumn(new FieldSchema("singlecolchainfield", "string", "cubecity.name"), + "cubecityname", "cubecity.name")); exprs.add(new ExprColumn(new FieldSchema("summsrs", "double", "sum measures"), "Sum Msrs", "(1000 + sum(msr1) + sum(msr2))/100")); exprs.add(new ExprColumn(new FieldSchema("msr5", "double", "materialized in some facts"), "Fifth Msr", http://git-wip-us.apache.org/repos/asf/lens/blob/38288194/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java index d54e8b9..f2bb485 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java @@ -440,4 +440,140 @@ public class TestExpressionResolver extends TestQueryRewrite { TestCubeRewriter.compareQueries(hqlQuery, expected); } + @Test + public void testSingleColExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolmsr2expr from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleDimColExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecoldim1expr from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct testcube.dim1 FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "c1_summary1")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleDimColExpressionWithAlias() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecoldim1expr as x from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct testcube.dim1 as x FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "C1_summary1")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleDimColQualifiedExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecoldim1qualifiedexpr from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct testcube.dim1 FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "C1_summary1")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleChainIdExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolchainid from testCube where " + TWO_DAYS_RANGE_IT, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct dim3chain.id FROM ", + " join " + getDbName() + "c2_testdim3tbl dim3chain on testcube.testdim3id = dim3chain.id", + null, null, null, + getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleChainRefIdExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolchainrefexpr from testCube where " + TWO_DAYS_RANGE_IT, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct testcube.testdim3id FROM ", null, null, + getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleChainRefColExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolchainfield from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct cubecity.name FROM ", + " join " + getDbName() + "c2_citytable cubecity ON testcube.cityid = cubecity.id", + null, null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleChainRefColExpressionWithAlias() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolchainfield as cityname from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select distinct cubecity.name as cityname FROM ", + " join " + getDbName() + "c2_citytable cubecity ON testcube.cityid = cubecity.id", + null, null, null, getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleColExpressionWithAlias() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolmsr2expr as msr2 from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select sum(testcube.msr2) msr2 FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleColQualifiedExpression() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolmsr2qualifiedexpr from testCube where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, + getWhereForDailyAndHourly2days(cubeName, "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + + @Test + public void testSingleColQualifiedExpressionWithAlias() throws Exception { + Configuration tconf = new Configuration(conf); + tconf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); + CubeQueryContext rewrittenQuery = + rewriteCtx("select singlecolmsr2qualifiedexpr from testCube tc where " + TWO_DAYS_RANGE, tconf); + String expected = + getExpectedQuery("tc", "select sum(tc.msr2) FROM ", null, null, + getWhereForDailyAndHourly2days("tc", "C2_testfact")); + TestCubeRewriter.compareQueries(rewrittenQuery.toHQL(), expected); + } + }
