LENS-1273 : Resolve issues with case when aggregate expressions with dim-attributes conditions
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2cfb7b09 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2cfb7b09 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2cfb7b09 Branch: refs/heads/master Commit: 2cfb7b097c5367f21779f3c27bc347a9ff536de1 Parents: 9ef7ce7 Author: Amareshwari Sriramadasu <[email protected]> Authored: Mon Oct 10 15:09:13 2016 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Mon Oct 10 15:09:13 2016 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/AggregateResolver.java | 24 --- .../apache/lens/cube/parse/AliasReplacer.java | 75 ++++---- .../apache/lens/cube/parse/CandidateFact.java | 2 +- .../lens/cube/parse/CandidateTableResolver.java | 161 +++++++++------- .../apache/lens/cube/parse/ColumnResolver.java | 110 +++++++---- .../lens/cube/parse/CubeQueryContext.java | 123 ++++-------- .../cube/parse/DenormalizationResolver.java | 4 +- .../lens/cube/parse/ExpressionResolver.java | 16 +- .../apache/lens/cube/parse/GroupbyResolver.java | 111 +++-------- .../lens/cube/parse/MultiFactHQLContext.java | 9 +- .../lens/cube/parse/QueriedPhraseContext.java | 186 +++++++++++++++++++ .../lens/cube/parse/SelectPhraseContext.java | 51 +++++ .../lens/cube/parse/TimeRangeChecker.java | 4 +- .../lens/cube/parse/TrackQueriedColumns.java | 29 ++- .../lens/cube/parse/TrackQueriedCubeFields.java | 66 +++++++ .../lens/cube/parse/TracksQueriedColumns.java | 59 ++++++ .../apache/lens/cube/parse/CubeTestSetup.java | 3 + .../lens/cube/parse/TestAggregateResolver.java | 62 +++---- .../lens/cube/parse/TestBaseCubeQueries.java | 115 +++++++++++- 19 files changed, 821 insertions(+), 389 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 292868a..c522061 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -159,15 +159,6 @@ class AggregateResolver implements ContextRewriter { if (wrapped != node) { if (parent != null) { parent.setChild(nodePos, wrapped); - // Check if this node has an alias - ASTNode sibling = HQLParser.findNodeByPath(parent, Identifier); - String expr; - if (sibling != null) { - expr = HQLParser.getString(parent); - } else { - expr = HQLParser.getString(wrapped); - } - cubeql.addAggregateExpr(expr.trim()); } else { return wrapped; } @@ -342,19 +333,4 @@ class AggregateResolver implements ContextRewriter { return false; } - - static void updateAggregates(ASTNode root, CubeQueryContext cubeql) { - if (root == null) { - return; - } - - if (HQLParser.isAggregateAST(root)) { - cubeql.addAggregateExpr(HQLParser.getString(root).trim()); - } else { - for (int i = 0; i < root.getChildCount(); i++) { - ASTNode child = (ASTNode) root.getChild(i); - updateAggregates(child, cubeql); - } - } - } } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java index 5b48ca4..da34242 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java @@ -18,10 +18,6 @@ */ package org.apache.lens.cube.parse; -import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_SELEXPR; - -import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -54,7 +50,11 @@ class AliasReplacer implements ContextRewriter { Map<String, String> colToTableAlias = cubeql.getColToTableAlias(); extractTabAliasForCol(cubeql); - findDimAttributesAndMeasures(cubeql); + // Resolve aliases in all queried phrases + for (QueriedPhraseContext qur : cubeql.getQueriedPhrases()) { + extractTabAliasForCol(colToTableAlias, qur); + } + findExpressionsAndMeasures(cubeql); if (colToTableAlias.isEmpty()) { return; @@ -83,11 +83,6 @@ class AliasReplacer implements ContextRewriter { replaceAliases(cubeql.getJoinAST(), 0, colToTableAlias); - // Update the aggregate expression set - AggregateResolver.updateAggregates(cubeql.getSelectAST(), cubeql); - AggregateResolver.updateAggregates(cubeql.getHavingAST(), cubeql); - // Update alias map as well - updateAliasMap(cubeql.getSelectAST(), cubeql); } /** @@ -95,27 +90,26 @@ class AliasReplacer implements ContextRewriter { * @param cubeql * @throws LensException */ - private void findDimAttributesAndMeasures(CubeQueryContext cubeql) throws LensException { + private void findExpressionsAndMeasures(CubeQueryContext cubeql) throws LensException { CubeInterface cube = cubeql.getCube(); if (cube != null) { - Set<String> cubeColsQueried = cubeql.getColumnsQueried(cube.getName()); - Set<String> queriedDimAttrs = new HashSet<String>(); - Set<String> queriedMsrs = new HashSet<String>(); - Set<String> queriedExprs = new HashSet<String>(); - if (cubeColsQueried != null && !cubeColsQueried.isEmpty()) { - for (String col : cubeColsQueried) { - if (cube.getMeasureNames().contains(col)) { - queriedMsrs.add(col); - } else if (cube.getDimAttributeNames().contains(col)) { - queriedDimAttrs.add(col); - } else if (cube.getExpressionNames().contains(col)) { - queriedExprs.add(col); + String cubeAlias = cubeql.getAliasForTableName(cube.getName()); + for (QueriedPhraseContext qur : cubeql.getQueriedPhrases()) { + Set<String> cubeColsQueried = qur.getColumnsQueried(cubeAlias); + if (cubeColsQueried != null && !cubeColsQueried.isEmpty()) { + for (String col : cubeColsQueried) { + if (cube.getMeasureNames().contains(col)) { + qur.addQueriedMsr(col); + } else if (cube.getDimAttributeNames().contains(col)) { + qur.addQueriedDimAttr(col); + } else if (cube.getExpressionNames().contains(col)) { + qur.addQueriedExprColumn(col); + } } } + cubeql.addQueriedMsrs(qur.getQueriedMsrs()); + cubeql.addQueriedExprs(qur.getQueriedExprColumns()); } - cubeql.addQueriedDimAttrs(queriedDimAttrs); - cubeql.addQueriedMsrs(queriedMsrs); - cubeql.addQueriedExprs(queriedExprs); } } @@ -164,6 +158,18 @@ class AliasReplacer implements ContextRewriter { } } + static void extractTabAliasForCol(Map<String, String> colToTableAlias, TrackQueriedColumns tqc) throws LensException { + Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE); + if (columns == null) { + return; + } + for (String col : columns) { + tqc.addColumnsQueried(colToTableAlias.get(col.toLowerCase()), col.toLowerCase()); + if (colToTableAlias.get(col.toLowerCase()) == null) { + throw new LensException(LensCubeErrorCode.COLUMN_NOT_FOUND.getLensErrorInfo(), col); + } + } + } static ASTNode replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) { if (node == null) { return node; @@ -213,21 +219,4 @@ class AliasReplacer implements ContextRewriter { return node; } - static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) { - if (root == null) { - return; - } - - if (root.getToken().getType() == TOK_SELEXPR) { - ASTNode alias = HQLParser.findNodeByPath(root, Identifier); - if (alias != null) { - cubeql.addExprToAlias(root, alias); - } - } else { - for (int i = 0; i < root.getChildCount(); i++) { - updateAliasMap((ASTNode) root.getChild(i), cubeql); - } - } - } - } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 01265a5..5dc9dc9 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -193,7 +193,7 @@ public class CandidateFact implements CandidateTable, QueryAST { dimFieldIndices.add(i); } ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); - String alias = cubeql.getSelectAlias(i); + String alias = cubeql.getSelectPhrases().get(i).getSelectAlias(); if (aliasNode != null) { String queryAlias = aliasNode.getText(); if (!queryAlias.equals(alias)) { http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 83e5088..510bd8c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -52,7 +52,7 @@ class CandidateTableResolver implements ContextRewriter { private boolean checkForQueriedColumns = true; - public CandidateTableResolver(Configuration conf) { + public CandidateTableResolver(Configuration ignored) { } @Override @@ -233,10 +233,17 @@ class CandidateTableResolver implements ContextRewriter { String str = cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName())); List<String> validFactTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); - Set<String> queriedDimAttrs = cubeql.getQueriedDimAttrs(); - Set<String> queriedMsrs = cubeql.getQueriedMsrs(); - // Remove fact tables based on columns in the query + Set<QueriedPhraseContext> queriedMsrs = new HashSet<>(); + Set<QueriedPhraseContext> dimExprs = new HashSet<>(); + for (QueriedPhraseContext qur : cubeql.getQueriedPhrases()) { + if (qur.hasMeasures(cubeql)) { + queriedMsrs.add(qur); + } else { + dimExprs.add(qur); + } + } + // Remove fact tables based on whether they are valid or not. for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { CandidateFact cfact = i.next(); @@ -249,26 +256,37 @@ class CandidateTableResolver implements ContextRewriter { continue; } } + + // update expression evaluability for this fact + for (String expr : cubeql.getQueriedExprs()) { + cubeql.getExprCtx().updateEvaluables(expr, cfact); + } + // go over the columns accessed in the query and find out which tables // can answer the query // the candidate facts should have all the dimensions queried and // atleast // one measure boolean toRemove = false; - for (String col : queriedDimAttrs) { - if (!cfact.getColumns().contains(col.toLowerCase())) { - // check if it available as reference, if not remove the candidate - if (!cubeql.getDeNormCtx().addRefUsage(cfact, col, cubeql.getCube().getName())) { - log.info("Not considering fact table:{} as column {} is not available", cfact, col); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(col)); - toRemove = true; - break; - } - } else if (!isFactColumnValidForRange(cubeql, cfact, col)) { + for (QueriedPhraseContext qur : dimExprs) { + if (!qur.isEvaluable(cubeql, cfact)) { + log.info("Not considering fact table:{} as columns {} are not available", cfact, qur.getColumns()); + cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(qur.getColumns())); toRemove = true; break; } } + + // check if the candidate fact has atleast one measure queried + // if expression has measures, they should be considered along with other measures and see if the fact can be + // part of measure covering set + if (!checkForFactColumnExistsAndValidForRange(cfact, queriedMsrs, cubeql)) { + Set<String> columns = getColumns(queriedMsrs); + + log.info("Not considering fact table:{} as columns {} is not available", cfact, columns); + cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(columns)); + toRemove = true; + } // go over join chains and prune facts that dont have any of the columns in each chain for (JoinChain chain : cubeql.getJoinchains().values()) { OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension)cubeql.getCubeTbls() @@ -284,45 +302,17 @@ class CandidateTableResolver implements ContextRewriter { } } } - // go over expressions queried - // if expression has no measures, prune facts which cannot evaluate expression - // if expression has measures, they should be considered along with other measures and see if the fact can be - // part of measure covering set - for (String expr : cubeql.getQueriedExprs()) { - cubeql.getExprCtx().updateEvaluables(expr, cfact); - if (!cubeql.getQueriedExprsWithMeasures().contains(expr) && !cubeql.getExprCtx().isEvaluable(expr, cfact)) { - // if expression has no measures, prune facts which cannot evaluate expression - log.info("Not considering fact table:{} as expression {} is not evaluatable", cfact, expr); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.expressionNotEvaluable(expr)); - toRemove = true; - break; - } - } - // check if the candidate fact has atleast one measure queried - // if expression has measures, they should be considered along with other measures and see if the fact can be - // part of measure covering set - if (!checkForFactColumnExistsAndValidForRange(cfact, queriedMsrs, cubeql) - && (cubeql.getQueriedExprsWithMeasures().isEmpty() - || cubeql.getExprCtx().allNotEvaluable(cubeql.getQueriedExprsWithMeasures(), cfact))) { - log.info("Not considering fact table:{} as columns {},{} is not available", cfact, queriedMsrs, - cubeql.getQueriedExprsWithMeasures()); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(queriedMsrs, - cubeql.getQueriedExprsWithMeasures())); - toRemove = true; - } + if (toRemove) { i.remove(); } } - Set<String> dimExprs = new HashSet<>(cubeql.getQueriedExprs()); - dimExprs.removeAll(cubeql.getQueriedExprsWithMeasures()); if (cubeql.getCandidateFacts().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), - (!queriedDimAttrs.isEmpty() ? queriedDimAttrs.toString() : "") - + (!dimExprs.isEmpty() ? dimExprs.toString() : "")); + getColumns(cubeql.getQueriedPhrases()).toString()); } Set<Set<CandidateFact>> cfactset; - if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) { + if (queriedMsrs.isEmpty()) { // if no measures are queried, add all facts individually as single covering sets cfactset = new HashSet<>(); for (CandidateFact cfact : cubeql.getCandidateFacts()) { @@ -334,18 +324,16 @@ class CandidateTableResolver implements ContextRewriter { } else { // Find out candidate fact table sets which contain all the measures // queried + List<CandidateFact> cfacts = new ArrayList<>(cubeql.getCandidateFacts()); - cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs, - cubeql.getQueriedExprsWithMeasures()); + cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs); log.info("Measure covering fact sets :{}", cfactset); - String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "") - + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : ""); + String msrString = getColumns(queriedMsrs).toString(); if (cfactset.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), msrString); } cubeql.getCandidateFactSets().addAll(cfactset); - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs, - cubeql.getQueriedExprsWithMeasures())); + cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(getColumns(queriedMsrs))); if (cubeql.getCandidateFacts().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), msrString); @@ -354,19 +342,25 @@ class CandidateTableResolver implements ContextRewriter { } } + private static Set<String> getColumns(Collection<QueriedPhraseContext> queriedPhraseContexts) { + Set<String> cols = new HashSet<>(); + for (QueriedPhraseContext qur : queriedPhraseContexts) { + cols.addAll(qur.getColumns()); + } + return cols; + } static Set<Set<CandidateFact>> findCoveringSets(CubeQueryContext cubeql, List<CandidateFact> cfactsPassed, - Set<String> msrs, Set<String> exprsWithMeasures) { + Set<QueriedPhraseContext> msrs) throws LensException { Set<Set<CandidateFact>> cfactset = new HashSet<>(); List<CandidateFact> cfacts = new ArrayList<>(cfactsPassed); for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) { CandidateFact cfact = i.next(); i.remove(); - // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable. - if ((msrs.isEmpty() || !checkForFactColumnExistsAndValidForRange(cfact, msrs, cubeql)) - && (exprsWithMeasures.isEmpty() || cubeql.getExprCtx().allNotEvaluable(exprsWithMeasures, cfact))) { + if (!checkForFactColumnExistsAndValidForRange(cfact, msrs, cubeql)) { + // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable. // ignore the fact continue; - } else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) { + } else if (allEvaluable(cfact, msrs, cubeql)) { // return single set Set<CandidateFact> one = new LinkedHashSet<>(); one.add(cfact); @@ -374,18 +368,18 @@ class CandidateTableResolver implements ContextRewriter { } else { // find the remaining measures in other facts if (i.hasNext()) { - Set<String> remainingMsrs = new HashSet<>(msrs); - Set<String> remainingExprs = new HashSet<>(exprsWithMeasures); - remainingMsrs.removeAll(cfact.getColumns()); - remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact)); - Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs); + Set<QueriedPhraseContext> remainingMsrs = new HashSet<>(msrs); + Set<QueriedPhraseContext> coveredMsrs = coveredMeasures(cfact, msrs, cubeql); + remainingMsrs.removeAll(coveredMsrs); + + Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs); if (!coveringSets.isEmpty()) { for (Set<CandidateFact> set : coveringSets) { set.add(cfact); cfactset.add(set); } } else { - log.info("Couldnt find any set containing remaining measures:{} {} in {}", remainingMsrs, remainingExprs, + log.info("Couldnt find any set containing remaining measures:{} {} in {}", remainingMsrs, cfactsPassed); } } @@ -682,8 +676,8 @@ class CandidateTableResolver implements ContextRewriter { // can answer the query for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) { CandidateDim cdim = i.next(); - if (cubeql.getColumnsQueried(dim.getName()) != null) { - for (String col : cubeql.getColumnsQueried(dim.getName())) { + if (cubeql.getColumnsQueriedForTable(dim.getName()) != null) { + for (String col : cubeql.getColumnsQueriedForTable(dim.getName())) { if (!cdim.getColumns().contains(col.toLowerCase())) { // check if the column is an expression if (cdim.getBaseTable().getExpressionNames().contains(col)) { @@ -711,7 +705,7 @@ class CandidateTableResolver implements ContextRewriter { if (cubeql.getCandidateDimTables().get(dim).size() == 0) { throw new LensException(LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo(), dim.getName(), cubeql - .getColumnsQueried(dim.getName()).toString()); + .getColumnsQueriedForTable(dim.getName()).toString()); } } } @@ -731,4 +725,41 @@ class CandidateTableResolver implements ContextRewriter { } return false; } + + static boolean checkForFactColumnExistsAndValidForRange(CandidateFact table, Collection<QueriedPhraseContext> colSet, + CubeQueryContext cubeql) throws LensException { + if (colSet == null || colSet.isEmpty()) { + return true; + } + for (QueriedPhraseContext qur : colSet) { + if (qur.isEvaluable(cubeql, table)) { + return true; + } + } + return false; + } + + static boolean allEvaluable(CandidateFact table, Collection<QueriedPhraseContext> colSet, + CubeQueryContext cubeql) throws LensException { + if (colSet == null || colSet.isEmpty()) { + return true; + } + for (QueriedPhraseContext qur : colSet) { + if (!qur.isEvaluable(cubeql, table)) { + return false; + } + } + return true; + } + + static Set<QueriedPhraseContext> coveredMeasures(CandidateFact table, Collection<QueriedPhraseContext> msrs, + CubeQueryContext cubeql) throws LensException { + Set<QueriedPhraseContext> coveringSet = new HashSet<>(); + for (QueriedPhraseContext msr : msrs) { + if (msr.isEvaluable(cubeql, table)) { + coveringSet.add(msr); + } + } + return coveringSet; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java index 2db5dd1..87e094a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java @@ -28,8 +28,11 @@ import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; import org.apache.lens.cube.parse.HQLParser.TreeNode; import org.apache.lens.server.api.error.LensException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; import com.google.common.base.Optional; @@ -40,10 +43,11 @@ class ColumnResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { + checkForAllColumnsSelected(cubeql); extractColumns(cubeql); } - private void extractColumns(CubeQueryContext cubeql) throws LensException { + private void checkForAllColumnsSelected(CubeQueryContext cubeql) throws LensException { // Check if its 'select * from...' ASTNode selTree = cubeql.getSelectAST(); if (selTree.getChildCount() == 1) { @@ -59,12 +63,15 @@ class ColumnResolver implements ContextRewriter { } } } + } + + private void extractColumns(CubeQueryContext cubeql) throws LensException { getColsForSelectTree(cubeql); getColsForWhereTree(cubeql); - getColsForTree(cubeql, cubeql.getJoinAST(), cubeql, true); - getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql, true); - getColsForTree(cubeql, cubeql.getHavingAST(), cubeql, true); - getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql, true); + getColsForAST(cubeql, cubeql.getJoinAST()); + getColsForAST(cubeql, cubeql.getGroupByAST()); + getColsForHavingAST(cubeql, cubeql.getHavingAST()); + getColsForAST(cubeql, cubeql.getOrderByAST()); // Update join dimension tables for (String table : cubeql.getTblAliasToColumns().keySet()) { @@ -76,6 +83,38 @@ class ColumnResolver implements ContextRewriter { } } + private void getColsForAST(CubeQueryContext cubeql, ASTNode clause) throws LensException { + if (clause == null) { + return; + } + for (int i = 0; i < clause.getChildCount(); i++) { + ASTNode queriedExpr = (ASTNode) clause.getChild(i); + QueriedPhraseContext qur = new QueriedPhraseContext(queriedExpr); + getColsForTree(cubeql, queriedExpr, qur, true); + cubeql.addColumnsQueried(qur.getTblAliasToColumns()); + cubeql.addQueriedPhrase(qur); + } + } + + private void getColsForHavingAST(CubeQueryContext cubeql, ASTNode clause) throws LensException { + if (clause == null) { + return; + } + + // split having clause phrases to be column level sothat having clause can be pushed to multiple facts if required. + if (HQLParser.isAggregateAST(clause) || clause.getType() == HiveParser.TOK_TABLE_OR_COL + || clause.getType() == HiveParser.DOT || clause.getChildCount() == 0) { + QueriedPhraseContext qur = new QueriedPhraseContext(clause); + qur.setAggregate(true); + getColsForTree(cubeql, clause, qur, true); + cubeql.addColumnsQueried(qur.getTblAliasToColumns()); + cubeql.addQueriedPhrase(qur); + } else { + for (Node child : clause.getChildren()) { + getColsForHavingAST(cubeql, (ASTNode)child); + } + } + } // finds columns in AST passed. static void getColsForTree(final CubeQueryContext cubeql, ASTNode tree, final TrackQueriedColumns tqc, final boolean skipAliases) @@ -99,7 +138,7 @@ class ColumnResolver implements ContextRewriter { // Take child ident.totext ASTNode ident = (ASTNode) node.getChild(0); String column = ident.getText().toLowerCase(); - if (skipAliases && cubeql.getExprToAliasMap().values().contains(column)) { + if (skipAliases && cubeql.isColumnAnAlias(column)) { // column is an existing alias return; } @@ -129,7 +168,13 @@ class ColumnResolver implements ContextRewriter { if (cubeql.getWhereAST() == null) { return; } - addColumnsForWhere(cubeql, cubeql.getWhereAST(), null); + for (int i = 0; i < cubeql.getWhereAST().getChildCount(); i++) { + ASTNode queriedExpr = (ASTNode) cubeql.getWhereAST().getChild(i); + QueriedPhraseContext qur = new QueriedPhraseContext(queriedExpr); + addColumnsForWhere(cubeql, qur, queriedExpr, cubeql.getWhereAST()); + cubeql.addColumnsQueried(qur.getTblAliasToColumns()); + cubeql.addQueriedPhrase(qur); + } } // Find all columns of select tree. @@ -153,13 +198,13 @@ class ColumnResolver implements ContextRewriter { ASTNode selectExpr = (ASTNode) cubeql.getSelectAST().getChild(i); ASTNode selectExprChild = (ASTNode)selectExpr.getChild(0); Set<String> cols = new HashSet<>(); - addColumnsForSelectExpr(cubeql, selectExpr, cubeql.getSelectAST(), cols); - ASTNode alias = HQLParser.findNodeByPath(selectExpr, Identifier); + SelectPhraseContext sel = new SelectPhraseContext(selectExpr); + addColumnsForSelectExpr(sel, selectExpr, cubeql.getSelectAST(), cols); + String alias = selectExpr.getChildCount() > 1 ? selectExpr.getChild(1).getText() : null; String selectAlias; String selectFinalAlias = null; if (alias != null) { - cubeql.addExprToAlias(selectExpr, alias); - selectFinalAlias = alias.getText(); + selectFinalAlias = alias; selectAlias = SELECT_ALIAS_PREFIX + exprInd; } else if (cols.size() == 1 && (selectExprChild.getToken().getType() == TOK_TABLE_OR_COL || selectExprChild.getToken().getType() == DOT)) { @@ -170,21 +215,26 @@ class ColumnResolver implements ContextRewriter { selectFinalAlias = HQLParser.getString(selectExprChild); } exprInd++; - cubeql.addSelectAlias(selectAlias, selectFinalAlias); + cubeql.addColumnsQueried(sel.getTblAliasToColumns()); + sel.setSelectAlias(selectAlias); + sel.setFinalAlias(!StringUtils.isBlank(selectFinalAlias) ? "`" + selectFinalAlias + "`" : selectAlias); + sel.setActualAlias(alias != null ? alias.toLowerCase() : null); + cubeql.addSelectPhrase(sel); } } - private static void addColumnsForWhere(final CubeQueryContext cubeql, ASTNode node, ASTNode parent) { + private static void addColumnsForWhere(final CubeQueryContext cubeql, QueriedPhraseContext qur, ASTNode node, + ASTNode parent) { if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) { // Take child ident.totext ASTNode ident = (ASTNode) node.getChild(0); String column = ident.getText().toLowerCase(); - if (cubeql.getExprToAliasMap().values().contains(column)) { + if (cubeql.isColumnAnAlias(column)) { // column is an existing alias return; } - addColumnQueriedWithTimeRangeFuncCheck(cubeql, parent, CubeQueryContext.DEFAULT_TABLE, column); + addColumnQueriedWithTimeRangeFuncCheck(cubeql, qur, parent, CubeQueryContext.DEFAULT_TABLE, column); } else if (node.getToken().getType() == DOT) { // This is for the case where column name is prefixed by table name @@ -197,32 +247,31 @@ class ColumnResolver implements ContextRewriter { String column = colIdent.getText().toLowerCase(); String table = tabident.getText().toLowerCase(); - addColumnQueriedWithTimeRangeFuncCheck(cubeql, parent, table, column); + addColumnQueriedWithTimeRangeFuncCheck(cubeql, qur, parent, table, column); } else if (node.getToken().getType() == TOK_FUNCTION) { ASTNode fname = HQLParser.findNodeByPath(node, Identifier); if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) { - addColumnsForWhere(cubeql, (ASTNode) node.getChild(1), node); + addColumnsForWhere(cubeql, qur, (ASTNode) node.getChild(1), node); } else { for (int i = 0; i < node.getChildCount(); i++) { - addColumnsForWhere(cubeql, (ASTNode) node.getChild(i), node); + addColumnsForWhere(cubeql, qur, (ASTNode) node.getChild(i), node); } } } else { for (int i = 0; i < node.getChildCount(); i++) { - addColumnsForWhere(cubeql, (ASTNode) node.getChild(i), node); + addColumnsForWhere(cubeql, qur, (ASTNode) node.getChild(i), node); } } } - private static void addColumnQueriedWithTimeRangeFuncCheck(final CubeQueryContext cubeql, final ASTNode parent, - final String table, final String column) { - + private static void addColumnQueriedWithTimeRangeFuncCheck(final CubeQueryContext cubeql, QueriedPhraseContext qur, + final ASTNode parent, final String table, final String column) { if (isTimeRangeFunc(parent)) { cubeql.addQueriedTimeDimensionCols(column); - cubeql.addColumnsQueriedWithTimeDimCheck(CubeQueryContext.DEFAULT_TABLE, column); + cubeql.addColumnsQueriedWithTimeDimCheck(qur, CubeQueryContext.DEFAULT_TABLE, column); } else { - cubeql.addColumnsQueried(table, column); + qur.addColumnsQueried(table, column); } } @@ -244,17 +293,14 @@ class ColumnResolver implements ContextRewriter { } return Optional.fromNullable(funcName); } - private static void addColumnsForSelectExpr(final CubeQueryContext cubeql, ASTNode node, ASTNode parent, + + private static void addColumnsForSelectExpr(final TrackQueriedColumns sel, ASTNode node, ASTNode parent, Set<String> cols) { if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) { // Take child ident.totext ASTNode ident = (ASTNode) node.getChild(0); String column = ident.getText().toLowerCase(); - if (cubeql.getExprToAliasMap().values().contains(column)) { - // column is an existing alias - return; - } - cubeql.addColumnsQueried(CubeQueryContext.DEFAULT_TABLE, column); + sel.addColumnsQueried(CubeQueryContext.DEFAULT_TABLE, column); cols.add(column); } else if (node.getToken().getType() == DOT) { // This is for the case where column name is prefixed by table name @@ -266,11 +312,11 @@ class ColumnResolver implements ContextRewriter { String column = colIdent.getText().toLowerCase(); String table = tabident.getText().toLowerCase(); - cubeql.addColumnsQueried(table, column); + sel.addColumnsQueried(table, column); cols.add(column); } else { for (int i = 0; i < node.getChildCount(); i++) { - addColumnsForSelectExpr(cubeql, (ASTNode) node.getChild(i), node, cols); + addColumnsForSelectExpr(sel, (ASTNode) node.getChild(i), node, cols); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 63ec8b2..6d53d00 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -59,7 +59,7 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeQueryContext implements TrackQueriedColumns, QueryAST { +public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { public static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; public static final String DEFAULT_TABLE = "_default_"; @@ -86,7 +86,6 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { // Joinchains accessed in the query @Getter protected Map<String, JoinChain> joinchains = new HashMap<String, JoinChain>(); - private final Set<String> queriedDimAttrs = new HashSet<String>(); @Getter private final Set<String> queriedMsrs = new HashSet<String>(); @@ -112,20 +111,31 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { // Alias to table object mapping of tables accessed in this query @Getter - private final Map<String, AbstractCubeTable> cubeTbls = new HashMap<String, AbstractCubeTable>(); - // Alias name to fields queried - @Getter - private final Map<String, Set<String>> tblAliasToColumns = new HashMap<String, Set<String>>(); - // Mapping of an expression to its column alias in the query - @Getter - private final Map<String, String> exprToAlias = new HashMap<String, String>(); - @Getter - private final List<String> selectAliases = new ArrayList<String>(); + private final Map<String, AbstractCubeTable> cubeTbls = new HashMap<>(); + + void addSelectPhrase(SelectPhraseContext sel) { + selectPhrases.add(sel); + addQueriedPhrase(sel); + } + + boolean isColumnAnAlias(String col) { + for (SelectPhraseContext sel : selectPhrases) { + if (col.equals(sel.getActualAlias())) { + return true; + } + } + return false; + } + + void addQueriedPhrase(QueriedPhraseContext qur) { + queriedPhrases.add(qur); + } @Getter - private final List<String> selectFinalAliases = new ArrayList<String>(); - // All aggregate expressions in the query + private final List<SelectPhraseContext> selectPhrases = new ArrayList<>(); + @Getter - private final Set<String> aggregateExprs = new HashSet<String>(); + private final List<QueriedPhraseContext> queriedPhrases = new ArrayList<>(); + // Join conditions used in all join expressions @Getter private final Map<QBJoinTree, String> joinConds = new HashMap<QBJoinTree, String>(); @@ -203,11 +213,6 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { this.selectAST = qb.getParseInfo().getSelForClause(clauseName); } - for (ASTNode aggrTree : qb.getParseInfo().getAggregationExprsForClause(clauseName).values()) { - String aggr = HQLParser.getString(aggrTree); - aggregateExprs.add(aggr); - } - extractMetaTables(); } @@ -670,10 +675,6 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { return qb.getParseInfo().getJoinExpr(); } - public QBJoinTree getQBJoinTree() { - return qb.getQbJoinTree(); - } - public String getOrderByString() { if (orderByAST != null) { return HQLParser.getString(orderByAST); @@ -1037,25 +1038,15 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { return ParseUtils.findRootNonNullToken(tree); } - public Set<String> getColumnsQueried(String tblName) { - return tblAliasToColumns.get(getAliasForTableName(tblName)); + public Set<String> getColumnsQueriedForTable(String tblName) { + return getColumnsQueried(getAliasForTableName(tblName)); } - public void addColumnsQueriedWithTimeDimCheck(String alias, String timeDimColumn) { + public void addColumnsQueriedWithTimeDimCheck(QueriedPhraseContext qur, String alias, String timeDimColumn) { if (!shouldReplaceTimeDimWithPart()) { - addColumnsQueried(alias, timeDimColumn); - } - } - - public void addColumnsQueried(String alias, String column) { - - Set<String> cols = tblAliasToColumns.get(alias.toLowerCase()); - if (cols == null) { - cols = new LinkedHashSet<String>(); - tblAliasToColumns.put(alias.toLowerCase(), cols); + qur.addColumnsQueried(alias, timeDimColumn); } - cols.add(column); } public boolean isCubeMeasure(String col) { @@ -1116,32 +1107,16 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { return isCubeMeasure(msrname); } - public boolean isAggregateExpr(String expr) { - return aggregateExprs.contains(expr == null ? null : expr.toLowerCase()); - } - public boolean hasAggregates() { - return !aggregateExprs.isEmpty() || getExprCtx().hasAggregates(); - } - - public String getAlias(String expr) { - return exprToAlias.get(expr); - } - - public String getSelectAlias(int index) { - return selectAliases.get(index); - } - - public String getSelectFinalAlias(int index) { - return selectFinalAliases.get(index); - } - - public Map<String, String> getExprToAliasMap() { - return exprToAlias; - } - - public void addAggregateExpr(String expr) { - aggregateExprs.add(expr); + if (getExprCtx().hasAggregates()) { + return true; + } + for (QueriedPhraseContext qur : queriedPhrases) { + if (qur.isAggregate()) { + return true; + } + } + return false; } public void setJoinCond(QBJoinTree qb, String cond) { @@ -1168,19 +1143,6 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { return ""; } - public void addExprToAlias(ASTNode expr, ASTNode alias) { - exprToAlias.put(HQLParser.getString(expr).trim(), alias.getText().toLowerCase()); - } - - public void addSelectAlias(String alias, String spacedAlias) { - selectAliases.add(alias); - if (!StringUtils.isBlank(spacedAlias)) { - selectFinalAliases.add("`" + spacedAlias + "`"); - } else { - selectFinalAliases.add(alias); - } - } - public Set<Aliased<Dimension>> getOptionalDimensions() { return optionalDimensionMap.keySet(); } @@ -1230,17 +1192,6 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } } - /** - * @return the queriedDimAttrs - */ - public Set<String> getQueriedDimAttrs() { - return queriedDimAttrs; - } - - public void addQueriedDimAttrs(Set<String> dimAttrs) { - queriedDimAttrs.addAll(dimAttrs); - } - public void addQueriedMsrs(Set<String> msrs) { queriedMsrs.addAll(msrs); } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index ab1710d..40ed387 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -360,7 +360,7 @@ public class DenormalizationResolver implements ContextRewriter { } if (cubeql.getCandidateFacts().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), - cubeql.getColumnsQueried(cubeql.getCube().getName()).toString()); + cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()).toString()); } cubeql.pruneCandidateFactSet(CandidateTablePruneCode.COLUMN_NOT_FOUND); } @@ -382,7 +382,7 @@ public class DenormalizationResolver implements ContextRewriter { if (cubeql.getCandidateDimTables().get(dim).size() == 0) { throw new LensException(LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo(), - dim.toString(), cubeql.getColumnsQueried(dim.getName()).toString()); + dim.toString(), cubeql.getColumnsQueriedForTable(dim.getName()).toString()); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 5adea6c..60dacdb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -59,8 +59,7 @@ class ExpressionResolver implements ContextRewriter { private Set<ExprSpecContext> allExprs = new LinkedHashSet<ExprSpecContext>(); private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>(); @Getter - private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions = - new HashMap<CandidateTable, Set<ExprSpecContext>>(); + private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions = new HashMap<>(); private boolean hasMeasures = false; public boolean hasMeasures() { @@ -197,16 +196,13 @@ class ExpressionResolver implements ContextRewriter { } } - static class ExprSpecContext implements TrackQueriedColumns { + static class ExprSpecContext extends TracksQueriedColumns { private Set<ExprSpec> exprSpecs = new LinkedHashSet<>(); @Getter @Setter private ASTNode finalAST; @Getter private Set<Dimension> exprDims = new HashSet<>(); - // for each expression store alias to columns queried - @Getter - private Map<String, Set<String>> tblAliasToColumns = new HashMap<>(); ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException { // replaces table names in expression with aliases in the query @@ -224,14 +220,6 @@ class ExpressionResolver implements ContextRewriter { AliasReplacer.extractTabAliasForCol(cubeql, this); finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); } - public void addColumnsQueried(String alias, String column) { - Set<String> cols = tblAliasToColumns.get(alias.toLowerCase()); - if (cols == null) { - cols = new HashSet<String>(); - tblAliasToColumns.put(alias.toLowerCase(), cols); - } - cols.add(column); - } void resolveColumns(CubeQueryContext cubeql) throws LensException { // finds all columns and table aliases in the expression http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index 8beeb9d..26ae1e7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -21,7 +21,6 @@ package org.apache.lens.cube.parse; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import org.apache.lens.cube.metadata.AbstractBaseTable; @@ -54,7 +53,7 @@ class GroupbyResolver implements ContextRewriter { CubeQueryConfUtil.DEFAULT_ENABLE_GROUP_BY_TO_SELECT); } - private void promoteSelect(CubeQueryContext cubeql, List<String> nonMsrNonAggSelExprsWithoutAlias, + private void promoteSelect(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, List<String> groupByExprs) throws LensException { if (!selectPromotionEnabled) { return; @@ -68,10 +67,10 @@ class GroupbyResolver implements ContextRewriter { // each selected column, if it is not a cube measure, and does not have // aggregation on the column, then it is added to group by columns. if (cubeql.hasAggregates()) { - for (String expr : nonMsrNonAggSelExprsWithoutAlias) { - + for (SelectPhraseContext sel : selectExprs) { + String expr = sel.getExprWithoutAlias(); if (!groupByExprs.contains(expr)) { - if (!cubeql.isAggregateExpr(expr)) { + if (!sel.isAggregate()) { ASTNode exprAST = HQLParser.parseExpr(expr); ASTNode groupbyAST = cubeql.getGroupByAST(); if (!isConstantsUsed(exprAST)) { @@ -116,23 +115,21 @@ class GroupbyResolver implements ContextRewriter { return false; } - private void promoteGroupby(CubeQueryContext cubeql, List<String> selectExprs, List<String> groupByExprs) + private void promoteGroupby(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, + List<String> groupByExprs) throws LensException { if (!groupbyPromotionEnabled) { return; } - for (String expr : selectExprs) { - expr = getExpressionWithoutAlias(cubeql, expr); - if (!cubeql.isAggregateExpr(expr)) { - log.info("Not promoting groupby expression to select, since there are expression projected"); - return; - } + if (selectExprs.size() > 0) { + log.info("Not promoting groupby expression to select, since there are expression projected"); + return; } int index = 0; for (String expr : groupByExprs) { - if (!contains(cubeql, selectExprs, expr)) { + if (!contains(selectExprs, expr)) { ASTNode exprAST = HQLParser.parseExpr(expr); addChildAtIndex(index, cubeql.getSelectAST(), exprAST); index++; @@ -158,33 +155,21 @@ class GroupbyResolver implements ContextRewriter { public void rewriteContext(CubeQueryContext cubeql) throws LensException { // Process Aggregations by making sure that all group by keys are projected; // and all projection fields are added to group by keylist; - List<String> selectExprs = new ArrayList<String>(); - String[] sel = getExpressions(cubeql.getSelectAST(), cubeql).toArray(new String[]{}); - for (String s : sel) { - selectExprs.add(s.trim()); - } - List<String> groupByExprs = new ArrayList<String>(); + List<SelectPhraseContext> selectExprs = getSelectNonAggregateNonMeasureExpressions(cubeql); + List<String> groupByExprs = new ArrayList<>(); if (cubeql.getGroupByString() != null) { - String[] gby = getExpressions(cubeql.getGroupByAST(), cubeql).toArray(new String[]{}); + String[] gby = getGroupbyExpressions(cubeql.getGroupByAST()).toArray(new String[]{}); for (String g : gby) { groupByExprs.add(g.trim()); } } - promoteSelect(cubeql, getNonMsrNonAggSelExprsWithoutAlias(cubeql.getSelectAST(), cubeql), groupByExprs); + promoteSelect(cubeql, selectExprs, groupByExprs); promoteGroupby(cubeql, selectExprs, groupByExprs); } - private String getExpressionWithoutAlias(CubeQueryContext cubeql, String sel) { - String alias = cubeql.getAlias(sel); - if (alias != null) { - sel = sel.substring(0, (sel.length() - alias.length())).trim(); - } - return sel; - } - - private boolean contains(CubeQueryContext cubeql, List<String> selExprs, String expr) { - for (String sel : selExprs) { - sel = getExpressionWithoutAlias(cubeql, sel); + private boolean contains(List<SelectPhraseContext> selExprs, String expr) { + for (SelectPhraseContext selExpr : selExprs) { + String sel = selExpr.getExprWithoutAlias(); if (sel.equals(expr)) { return true; } @@ -192,72 +177,32 @@ class GroupbyResolver implements ContextRewriter { return false; } - /** - * @param selectASTNode a select AST Node - * @param cubeQueryCtx - * @return List of non measure and non aggregate select expressions in string format without aliases - */ - private List<String> getNonMsrNonAggSelExprsWithoutAlias(final ASTNode selectASTNode, CubeQueryContext cubeQueryCtx) { - - List<String> nonMsrNonAggSelExprsWithoutAlias = new LinkedList<String>(); - List<ASTNode> nonMsrNonAggSelASTChildren = filterNonMsrNonAggSelectASTChildren(selectASTNode, cubeQueryCtx); - - for (ASTNode nonMsrNonAggSelASTChild : nonMsrNonAggSelASTChildren) { + private List<SelectPhraseContext> getSelectNonAggregateNonMeasureExpressions(CubeQueryContext cubeql) { - /* Assuming all children of SelectASTNode are SELECT Expression AST Nodes only. - Refer:https://reviews.apache.org/r/29422/#comment109498 for more details. - Order of Children of select expression AST Node => Index 0: Select Expression Without Alias, Index 1: Alias */ + List<SelectPhraseContext> list = new ArrayList<>(); - ASTNode selExprWithoutAlias = (ASTNode) nonMsrNonAggSelASTChild.getChildren().get(0); - String result = HQLParser.getString(selExprWithoutAlias); - nonMsrNonAggSelExprsWithoutAlias.add(result); - - } - return nonMsrNonAggSelExprsWithoutAlias; - } - - /** - * @param selectASTNode a select ASTNode - * @param cubeQueryCtx - * @return list of selectASTNode Children which does not contain a measure or an aggregate. Empty list is returned - * when selectASTNode is not a Select AST Node. Empty list is returned when there are no non measure and non aggregate - * children nodes present in select AST. - */ - private List<ASTNode> filterNonMsrNonAggSelectASTChildren(final ASTNode selectASTNode, - CubeQueryContext cubeQueryCtx) { - List<ASTNode> nonMsrNonAggSelASTChildren = new LinkedList<ASTNode>(); - - if (!HQLParser.isSelectASTNode(selectASTNode)) { - return nonMsrNonAggSelASTChildren; - } - - for (int i = 0; i < selectASTNode.getChildCount(); i++) { - ASTNode childNode = (ASTNode) selectASTNode.getChild(i); - if (hasMeasure(childNode, cubeQueryCtx) || hasAggregate(childNode, cubeQueryCtx)) { + for (SelectPhraseContext sel : cubeql.getSelectPhrases()) { + if (hasMeasure(sel.getExprAST(), cubeql)) { + continue; + } + if (hasAggregate(sel.getExprAST(), cubeql)) { continue; } - nonMsrNonAggSelASTChildren.add(childNode); + list.add(sel); } - return nonMsrNonAggSelASTChildren; + return list; } - private List<String> getExpressions(ASTNode node, CubeQueryContext cubeql) { + private List<String> getGroupbyExpressions(ASTNode node) { - List<String> list = new ArrayList<String>(); + List<String> list = new ArrayList<>(); if (node == null) { return list; } for (int i = 0; i < node.getChildCount(); i++) { - ASTNode child = (ASTNode) node.getChild(i); - if (hasMeasure(child, cubeql)) { - continue; - } - if (hasAggregate(child, cubeql)) { - continue; - } list.add(HQLParser.getString((ASTNode) node.getChild(i))); } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java index b3547db..979c24b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java @@ -111,17 +111,18 @@ class MultiFactHQLContext extends SimpleHQLContext { } if (selectToFactIndex.get(i).size() == 1) { select.append("mq").append(selectToFactIndex.get(i).get(0)).append(".") - .append(query.getSelectAlias(i)).append(" "); + .append(query.getSelectPhrases().get(i).getSelectAlias()).append(" "); } else { select.append("COALESCE("); String sep = ""; for (Integer factIndex : selectToFactIndex.get(i)) { - select.append(sep).append("mq").append(factIndex).append(".").append(query.getSelectAlias(i)); + select.append(sep).append("mq").append(factIndex).append(".").append( + query.getSelectPhrases().get(i).getSelectAlias()); sep = ", "; } select.append(") "); } - select.append(query.getSelectFinalAlias(i)); + select.append(query.getSelectPhrases().get(i).getFinalAlias()); if (i != query.getSelectAST().getChildCount() - 1) { select.append(", "); } @@ -152,7 +153,7 @@ class MultiFactHQLContext extends SimpleHQLContext { fromBuilder.append(" on "); Iterator<Integer> dimIter = fact.getDimFieldIndices().iterator(); while (dimIter.hasNext()) { - String dim = query.getSelectAlias(dimIter.next()); + String dim = query.getSelectPhrases().get(dimIter.next()).getSelectAlias(); fromBuilder.append(getMultiFactJoinCondition(aliasCount, dim)); if (dimIter.hasNext()) { fromBuilder.append(" AND "); http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java new file mode 100644 index 0000000..11eb8f7 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.*; + +import org.apache.lens.cube.metadata.MetastoreConstants; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.parse.ASTNode; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +@Data +@EqualsAndHashCode(callSuper = true) +@Slf4j +class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedCubeFields { + private final ASTNode exprAST; + private Boolean aggregate; + private String expr; + private final Set<String> queriedDimAttrs = new HashSet<>(); + private final Set<String> queriedMsrs = new HashSet<>(); + private final Set<String> queriedExprColumns = new HashSet<>(); + private final Set<String> columns = new HashSet<>(); + + void setNotAggregate() { + this.aggregate = false; + } + + boolean isAggregate() { + if (aggregate == null) { + aggregate = HQLParser.hasAggregate(exprAST); + } + return aggregate; + } + + String getExpr() { + if (expr == null) { + expr = HQLParser.getString(getExprAST()).trim(); + } + return expr; + } + + void updateExprs() { + expr = HQLParser.getString(getExprAST()).trim(); + } + + @Override + public void addQueriedDimAttr(String attrName) { + queriedDimAttrs.add(attrName); + columns.add(attrName); + } + + @Override + public void addQueriedMsr(String msrName) { + queriedMsrs.add(msrName); + columns.add(msrName); + } + + @Override + public void addQueriedExprColumn(String exprCol) { + queriedExprColumns.add(exprCol); + columns.add(exprCol); + } + + public boolean hasMeasures(CubeQueryContext cubeQl) { + if (!queriedMsrs.isEmpty()) { + return true; + } + if (!queriedExprColumns.isEmpty()) { + for (String exprCol : queriedExprColumns) { + if (cubeQl.getQueriedExprsWithMeasures().contains(exprCol)) { + return true; + } + } + } + return false; + } + + boolean isEvaluable(CubeQueryContext cubeQl, CandidateFact cfact) throws LensException { + // all measures of the queried phrase should be present + for (String msr : queriedMsrs) { + if (!checkForColumnExistsAndValidForRange(cfact, msr, cubeQl)) { + return false; + } + } + // all expression columns should be evaluable + for (String exprCol : queriedExprColumns) { + if (!cubeQl.getExprCtx().isEvaluable(exprCol, cfact)) { + log.info("expression {} is not evaluable in fact table:{}", expr, cfact); + return false; + } + } + // all dim-attributes should be present. + for (String col : queriedDimAttrs) { + if (!cfact.getColumns().contains(col.toLowerCase())) { + // check if it available as reference + if (!cubeQl.getDeNormCtx().addRefUsage(cfact, col, cubeQl.getCube().getName())) { + log.info("column {} is not available in fact table:{} ", col, cfact); + return false; + } + } else if (!isFactColumnValidForRange(cubeQl, cfact, col)) { + log.info("column {} is not available in range queried in fact {}", col, cfact); + return false; + } + } + return true; + } + + public static boolean isColumnAvailableInRange(final TimeRange range, Date startTime, Date endTime) { + return (isColumnAvailableFrom(range.getFromDate(), startTime) + && isColumnAvailableTill(range.getToDate(), endTime)); + } + + public static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { + return (startTime == null) || date.equals(startTime) || date.after(startTime); + } + + public static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { + return (endTime == null) || date.equals(endTime) || date.before(endTime); + } + + public static boolean isFactColumnValidForRange(CubeQueryContext cubeql, CandidateTable cfact, String col) { + for(TimeRange range : cubeql.getTimeRanges()) { + if (!isColumnAvailableInRange(range, getFactColumnStartTime(cfact, col), getFactColumnEndTime(cfact, col))) { + return false; + } + } + return true; + } + + public static Date getFactColumnStartTime(CandidateTable table, String factCol) { + Date startTime = null; + if (table instanceof CandidateFact) { + for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { + if (key.contains(MetastoreConstants.FACT_COL_START_TIME_PFX)) { + String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_START_TIME_PFX); + if (factCol.equals(propCol)) { + startTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); + } + } + } + } + return startTime; + } + + public static Date getFactColumnEndTime(CandidateTable table, String factCol) { + Date endTime = null; + if (table instanceof CandidateFact) { + for (String key : ((CandidateFact) table).fact.getProperties().keySet()) { + if (key.contains(MetastoreConstants.FACT_COL_END_TIME_PFX)) { + String propCol = StringUtils.substringAfter(key, MetastoreConstants.FACT_COL_END_TIME_PFX); + if (factCol.equals(propCol)) { + endTime = ((CandidateFact) table).fact.getDateFromProperty(key, false, true); + } + } + } + } + return endTime; + } + + static boolean checkForColumnExistsAndValidForRange(CandidateTable table, String column, CubeQueryContext cubeql) { + return (table.getColumns().contains(column) && isFactColumnValidForRange(cubeql, table, column)); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/SelectPhraseContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SelectPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SelectPhraseContext.java new file mode 100644 index 0000000..b6702e5 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SelectPhraseContext.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import org.apache.hadoop.hive.ql.parse.ASTNode; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +class SelectPhraseContext extends QueriedPhraseContext { + private String actualAlias; + private String selectAlias; + private String finalAlias; + private String exprWithoutAlias; + + public SelectPhraseContext(ASTNode selectExpr) { + super(selectExpr); + } + + String getExprWithoutAlias() { + if (exprWithoutAlias == null) { + //Order of Children of select expression AST Node => Index 0: Select Expression Without Alias, Index 1: Alias */ + exprWithoutAlias = HQLParser.getString((ASTNode) getExprAST().getChild(0)).trim(); + } + return exprWithoutAlias; + } + + void updateExprs() { + super.updateExprs(); + exprWithoutAlias = HQLParser.getString((ASTNode) getExprAST().getChild(0)).trim(); + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java index ca176ee..89b50f5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java @@ -139,13 +139,13 @@ public class TimeRangeChecker implements ContextRewriter { private void doColLifeValidation(CubeQueryContext cubeql) throws LensException, ColUnAvailableInTimeRangeException { - Set<String> cubeColumns = cubeql.getColumnsQueried(cubeql.getCube().getName()); + Set<String> cubeColumns = cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()); if (cubeColumns == null || cubeColumns.isEmpty()) { // Query doesn't have any columns from cube return; } - for (String col : cubeql.getColumnsQueried(cubeql.getCube().getName())) { + for (String col : cubeql.getColumnsQueriedForTable(cubeql.getCube().getName())) { CubeColumn column = cubeql.getCube().getColumnByName(col); for (TimeRange range : cubeql.getTimeRanges()) { if (column == null) { http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedColumns.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedColumns.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedColumns.java index b65ac26..45d59df 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedColumns.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedColumns.java @@ -22,6 +22,33 @@ import java.util.Map; import java.util.Set; interface TrackQueriedColumns { + + /** + * Get tblAlias to column + * @return map of tblAliasToColumns + */ Map<String, Set<String>> getTblAliasToColumns(); - void addColumnsQueried(String alias, String column); + + /** + * Get columns queried by tblAlias + * + * @param tblAlias tbl alias name + * + * @return set of column names + */ + Set<String> getColumnsQueried(String tblAlias); + + /** + * Add given table alias and column + * @param tblAlias Table alias + * @param column column + */ + void addColumnsQueried(String tblAlias, String column); + + /** + * Add given map of tblAliasToColumns. + * + * @param tblAliasToColumns map of tblAliasToColumns + */ + void addColumnsQueried(Map<String, Set<String>> tblAliasToColumns); } http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedCubeFields.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedCubeFields.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedCubeFields.java new file mode 100644 index 0000000..eafbb0e --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackQueriedCubeFields.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.Set; + +interface TrackQueriedCubeFields { + + /** + * Get queried dim attributes + * + * @return set of dim attribute names + */ + Set<String> getQueriedDimAttrs(); + + /** + * Get queried measures + * + * @return set of measure names + */ + Set<String> getQueriedMsrs(); + + /** + * Get queried expr columns + * + * @return set of expr column names + */ + Set<String> getQueriedExprColumns(); + + /** + * Add queried dim attribute + * + * @param attrName attribute name + */ + void addQueriedDimAttr(String attrName); + + /** + * Add queried measure + * + * @param msrName measure name + */ + void addQueriedMsr(String msrName); + + /** + * Add queried expression column + * + * @param exprCol expression column name + */ + void addQueriedExprColumn(String exprCol); +} http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/main/java/org/apache/lens/cube/parse/TracksQueriedColumns.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TracksQueriedColumns.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TracksQueriedColumns.java new file mode 100644 index 0000000..fb16478 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TracksQueriedColumns.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import lombok.Getter; + +abstract class TracksQueriedColumns implements TrackQueriedColumns { + + @Getter + private Map<String, Set<String>> tblAliasToColumns = new HashMap<>(); + + public void addColumnsQueried(String tblAlias, String column) { + + Set<String> cols = tblAliasToColumns.get(tblAlias.toLowerCase()); + if (cols == null) { + cols = new LinkedHashSet<>(); + tblAliasToColumns.put(tblAlias.toLowerCase(), cols); + } + cols.add(column); + } + + public void addColumnsQueried(Map<String, Set<String>> tblAliasToColumns) { + + for (Map.Entry<String, Set<String>> entry : tblAliasToColumns.entrySet()) { + Set<String> cols = this.tblAliasToColumns.get(entry.getKey()); + if (cols == null) { + cols = new LinkedHashSet<>(); + this.tblAliasToColumns.put(entry.getKey(), cols); + } + cols.addAll(entry.getValue()); + } + } + + public Set<String> getColumnsQueried(String tblAlias) { + return tblAliasToColumns.get(tblAlias); + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/2cfb7b09/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index f7f8af2..0c43cb5 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -984,6 +984,7 @@ public class CubeTestSetup { "dim2chain", "id", null, null, null)); // used as key in the chains cubeDimensions2.add(new ReferencedDimAttribute(new FieldSchema("dim22", "int", "ref dim"), "Dim2 refer", "dim2chain", "id", null, null, null)); // not used as key in the chains + cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim13", "string", "basedim"))); cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("userid", "int", "userid"))); cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("xuserid", "int", "userid"))); cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("yuserid", "int", "userid"))); @@ -1232,6 +1233,7 @@ public class CubeTestSetup { dimensions.add("dim2big2"); dimensions.add("dim2bignew"); dimensions.add("dim11"); + dimensions.add("dim13"); dimensions.add("dim12"); dimensions.add("dim22"); dimensions.add("d_time"); @@ -1408,6 +1410,7 @@ public class CubeTestSetup { factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time")); factColumns.add(new FieldSchema("dim1", "string", "base dim")); factColumns.add(new FieldSchema("dim11", "string", "base dim")); + factColumns.add(new FieldSchema("dim13", "string", "base dim")); factColumns.add(new FieldSchema("dim12", "string", "base dim")); factColumns.add(new FieldSchema("dim22", "string", "base dim")); factColumns.add(new FieldSchema("cityid", "int", "city id"));
