LENS-174: (incremental) Cube rewriter changes to support multiple expressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/099d19e8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/099d19e8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/099d19e8 Branch: refs/heads/current-release-line Commit: 099d19e881332b90eadc1fb36b443d2954c11bd2 Parents: ad471b2 Author: Amareshwari Sriramadasu <[email protected]> Authored: Tue Jun 9 12:16:01 2015 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Tue Jun 9 12:16:01 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/AggregateResolver.java | 6 +- .../apache/lens/cube/parse/AliasReplacer.java | 6 +- .../lens/cube/parse/CandidateTableResolver.java | 75 ++++++++++++-------- .../lens/cube/parse/CubeQueryContext.java | 15 ++-- .../lens/cube/parse/ExpressionResolver.java | 33 ++++----- 5 files changed, 75 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 2ad2b7c..edf0f74 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -218,7 +218,7 @@ class AggregateResolver implements ContextRewriter { if (node.getChild(0).getType() == HiveParser.Identifier) { function = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText()); } - } else if (cubeql.isMeasure(node)) { + } else if (cubeql.isCubeMeasure(node)) { // Exit for the recursion String colname; @@ -291,7 +291,7 @@ class AggregateResolver implements ContextRewriter { boolean isDistinct = hasDistinct; if (exprTokenType == HiveParser.TOK_FUNCTIONDI || exprTokenType == HiveParser.TOK_SELECTDI) { isDistinct = true; - } else if (cubeql.isMeasure(node) && isDistinct) { + } else if (cubeql.isCubeMeasure(node) && isDistinct) { // Exit for the recursion return true; } @@ -310,7 +310,7 @@ class AggregateResolver implements ContextRewriter { return false; } - if (cubeql.isMeasure(node)) { + if (cubeql.isCubeMeasure(node)) { return true; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java index 4d3443c..9309307 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java @@ -120,11 +120,11 @@ class AliasReplacer implements ContextRewriter { } private void extractTabAliasForCol(CubeQueryContext cubeql) throws SemanticException { - extractTabAliasForCol(cubeql.getColToTableAlias(), cubeql, cubeql); + extractTabAliasForCol(cubeql, cubeql); } - static void extractTabAliasForCol(Map<String, String> colToTableAlias, CubeQueryContext cubeql, - TrackQueriedColumns tqc) throws SemanticException { + static void extractTabAliasForCol(CubeQueryContext cubeql, TrackQueriedColumns tqc) throws SemanticException { + Map<String, String> colToTableAlias = cubeql.getColToTableAlias(); Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE); if (columns == null) { return; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index d56fb80..53fb11f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -284,23 +284,35 @@ class CandidateTableResolver implements ContextRewriter { (!queriedDimAttrs.isEmpty() ? queriedDimAttrs.toString() : "") + (!dimExprs.isEmpty() ? dimExprs.toString() : "")); } - // Find out candidate fact table sets which contain all the measures - // queried - List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts()); - Set<Set<CandidateFact>> cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs, - cubeql.getQueriedExprsWithMeasures()); - LOG.info("Measure covering fact sets :" + cfactset); - String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "") - + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : ""); - if (cfactset.isEmpty()) { - throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString); - } - cubeql.getCandidateFactSets().addAll(cfactset); - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs, - cubeql.getQueriedExprsWithMeasures())); + Set<Set<CandidateFact>> cfactset; + if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) { + // if no measures are queried, add all facts individually as single covering sets + cfactset = new HashSet<Set<CandidateFact>>(); + for (CandidateFact cfact : cubeql.getCandidateFacts()) { + Set<CandidateFact> one = new LinkedHashSet<CandidateFact>(); + one.add(cfact); + cfactset.add(one); + } + cubeql.getCandidateFactSets().addAll(cfactset); + } else { + // Find out candidate fact table sets which contain all the measures + // queried + List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts()); + cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs, + cubeql.getQueriedExprsWithMeasures()); + LOG.info("Measure covering fact sets :" + cfactset); + String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "") + + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : ""); + if (cfactset.isEmpty()) { + throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString); + } + cubeql.getCandidateFactSets().addAll(cfactset); + cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs, + cubeql.getQueriedExprsWithMeasures())); - if (cubeql.getCandidateFacts().size() == 0) { - throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString); + if (cubeql.getCandidateFacts().size() == 0) { + throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString); + } } } } @@ -312,10 +324,10 @@ class CandidateTableResolver implements ContextRewriter { for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) { CandidateFact cfact = i.next(); i.remove(); - if (!checkForColumnExists(cfact, msrs) + // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable. + if ((msrs.isEmpty() || !checkForColumnExists(cfact, msrs)) && (exprsWithMeasures.isEmpty() || cubeql.getExprCtx().allNotEvaluable(exprsWithMeasures, cfact))) { - // check if fact contains any of the maeasures - // if not ignore the fact + // ignore the fact continue; } else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) { // return single set @@ -324,18 +336,21 @@ class CandidateTableResolver implements ContextRewriter { cfactset.add(one); } else { // find the remaining measures in other facts - Set<String> remainingMsrs = new HashSet<String>(msrs); - Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures); - remainingMsrs.removeAll(cfact.getColumns()); - remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact)); - Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs); - if (!coveringSets.isEmpty()) { - for (Set<CandidateFact> set : coveringSets) { - set.add(cfact); - cfactset.add(set); + if (i.hasNext()) { + Set<String> remainingMsrs = new HashSet<String>(msrs); + Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures); + remainingMsrs.removeAll(cfact.getColumns()); + remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact)); + Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs); + if (!coveringSets.isEmpty()) { + for (Set<CandidateFact> set : coveringSets) { + set.add(cfact); + cfactset.add(set); + } + } else { + LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs + + " in " + cfactsPassed); } - } else { - LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs); } } } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 0409af3..c5d8d91 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.parse; import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; - import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE; import java.io.ByteArrayOutputStream; @@ -803,20 +802,20 @@ public class CubeQueryContext implements TrackQueriedColumns { } // pick dimension tables required during expression expansion for the picked fact and dimensions - Set<Dimension> exprDimTables = new HashSet<Dimension>(); + Set<Dimension> exprDimensions = new HashSet<Dimension>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1); - exprDimTables.addAll(factExprDimTables); + exprDimensions.addAll(factExprDimTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factExprDimTables); } } } else { // dim only query - exprDimTables.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false)); + exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false)); } - dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimTables)); + dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); // pick denorm tables for the picked fact and dimensions Set<Dimension> denormTables = new HashSet<Dimension>(); @@ -927,7 +926,7 @@ public class CubeQueryContext implements TrackQueriedColumns { if (split.length <= 1) { col = col.trim().toLowerCase(); if (queriedExprs.contains(col)) { - return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).isHasMeasures(); + return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).hasMeasures(); } else { return cube.getMeasureNames().contains(col); } @@ -936,7 +935,7 @@ public class CubeQueryContext implements TrackQueriedColumns { String colName = split[1].trim().toLowerCase(); if (cubeName.equalsIgnoreCase(cube.getName()) || cubeName.equals(getAliasForTableName(cube.getName()))) { if (queriedExprs.contains(colName)) { - return exprCtx.getExpressionContext(colName, cubeName).isHasMeasures(); + return exprCtx.getExpressionContext(colName, cubeName).hasMeasures(); } else { return cube.getMeasureNames().contains(colName.toLowerCase()); } @@ -946,7 +945,7 @@ public class CubeQueryContext implements TrackQueriedColumns { } } - boolean isMeasure(ASTNode node) { + boolean isCubeMeasure(ASTNode node) { String tabname = null; String colname; int nodeType = node.getToken().getType(); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index f2a7039..539badb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -70,9 +70,12 @@ class ExpressionResolver implements ContextRewriter { private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>(); private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions = new HashMap<CandidateTable, Set<ExprSpecContext>>(); - @Getter private boolean hasMeasures = false; + public boolean hasMeasures() { + return hasMeasures; + } + ExpressionContext(CubeQueryContext cubeql, ExprColumn exprCol, AbstractBaseTable srcTable, String srcAlias) throws SemanticException { this.srcTable = srcTable; @@ -82,13 +85,12 @@ class ExpressionResolver implements ContextRewriter { allExprs.add(new ExprSpecContext(es, cubeql)); } resolveColumnsAndAlias(cubeql); + log.debug("All exprs for {} are {}", exprCol.getName(), allExprs); } private void resolveColumnsAndAlias(CubeQueryContext cubeql) throws SemanticException { for (ExprSpecContext esc : allExprs) { esc.resolveColumns(cubeql); - esc.replaceAliasInAST(cubeql, cubeql.getColToTableAlias()); - } - for (ExprSpecContext esc : allExprs) { + esc.replaceAliasInAST(cubeql); for (String table : esc.getTblAliasToColumns().keySet()) { try { if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) { @@ -101,21 +103,18 @@ class ExpressionResolver implements ContextRewriter { } } } - resolveColumnsAndReplaceAlias(cubeql, allExprs, cubeql.getColToTableAlias()); + resolveColumnsAndReplaceAlias(cubeql, allExprs); } - private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs, - Map<String, String> colToTableAlias) throws SemanticException { + private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs) + throws SemanticException { Set<ExprSpecContext> nestedExpressions = new LinkedHashSet<ExprSpecContext>(); for (ExprSpecContext esc : exprs) { - for (Map.Entry<String, Set<String>> entry : esc.tblAliasToColumns.entrySet()) { + for (Map.Entry<String, Set<String>> entry : esc.getTblAliasToColumns().entrySet()) { if (entry.getKey().equals(CubeQueryContext.DEFAULT_TABLE)) { continue; } AbstractBaseTable baseTable = (AbstractBaseTable)cubeql.getCubeTableForAlias(entry.getKey()); - // if (baseTable == null) { - // continue; - // } Set<String> exprCols = new HashSet<String>(); for (String col : entry.getValue()) { // col is an expression @@ -129,11 +128,11 @@ class ExpressionResolver implements ContextRewriter { } for (ExprSpecContext esc : nestedExpressions) { esc.resolveColumns(cubeql); - esc.replaceAliasInAST(cubeql, colToTableAlias); + esc.replaceAliasInAST(cubeql); for (String table : esc.getTblAliasToColumns().keySet()) { try { if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) { - cubeql.addOptionalDimTable(table, null, true, + cubeql.addOptionalDimTable(table, null, false, esc.getTblAliasToColumns().get(table).toArray(new String[0])); esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table)); } @@ -217,10 +216,10 @@ class ExpressionResolver implements ContextRewriter { exprSpecs.add(current); finalAST = replaceAlias(node, cubeql); } - public void replaceAliasInAST(CubeQueryContext cubeql, Map<String, String> colToTableAlias) + public void replaceAliasInAST(CubeQueryContext cubeql) throws SemanticException { - AliasReplacer.extractTabAliasForCol(colToTableAlias, cubeql, this); - AliasReplacer.replaceAliases(finalAST, 0, colToTableAlias); + AliasReplacer.extractTabAliasForCol(cubeql, this); + AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); } public void addColumnsQueried(String alias, String column) { Set<String> cols = tblAliasToColumns.get(alias.toLowerCase()); @@ -368,12 +367,14 @@ class ExpressionResolver implements ContextRewriter { if (!cTable.getColumns().contains(col.toLowerCase())) { if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) { // check if it is available as reference, if not expression is not evaluable + log.debug("{} = {} is not evaluable in {}", expr, esc, cTable); isEvaluable = false; break; } } } if (isEvaluable) { + log.debug("{} = {} is evaluable in {}", expr, esc, cTable); ec.addEvaluable(cubeql, cTable, esc); } }
