LENS-1378 : Fix dimensions to query from DenormalizationResolver for expression fields
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fb25061 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fb25061 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fb25061 Branch: refs/heads/lens-1381 Commit: 4fb2506193acd3efa7d90bb640bd50e2aefe5d6a Parents: 98990c3 Author: Amareshwari Sriramadasu <amareshw...@apache.org> Authored: Wed Jan 18 17:46:58 2017 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Jan 18 17:46:58 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/CandidateTableResolver.java | 2 +- .../lens/cube/parse/CubeQueryContext.java | 11 +- .../cube/parse/DenormalizationResolver.java | 167 +++++++++++++------ .../lens/cube/parse/ExpressionResolver.java | 107 ++++++------ .../lens/cube/parse/QueriedPhraseContext.java | 2 +- .../lens/cube/parse/TimeRangeChecker.java | 18 -- .../lens/cube/parse/TrackDenormContext.java | 37 ++++ .../apache/lens/cube/parse/CubeTestSetup.java | 11 +- .../lens/cube/parse/TestExpressionResolver.java | 15 ++ .../lens/cube/parse/TestJoinResolver.java | 2 +- tools/conf/server/logback.xml | 4 +- 11 files changed, 248 insertions(+), 128 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index e7fc557..e9270ea 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -697,7 +697,7 @@ class CandidateTableResolver implements ContextRewriter { i.remove(); break; } - } else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) { + } else if (!cubeql.getDeNormCtx().addRefUsage(cubeql, cdim, col, dim.getName())) { // check if it available as reference, if not remove the // candidate log.info("Not considering dimtable: {} as column {} is not available", cdim, col); http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index e83ae76..125b432 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -59,7 +59,7 @@ import lombok.*; import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { +public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, TrackDenormContext { public static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; public static final String DEFAULT_TABLE = "_default_"; @@ -922,7 +922,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { Set<Dimension> exprDimensions = new HashSet<>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { - Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this); + Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(this, cfact, dimsToQuery, + cfacts.size() > 1 ? cfact : this); exprDimensions.addAll(factExprDimTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factExprDimTables); @@ -933,7 +934,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { } } else { // dim only query - exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this)); + exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); @@ -942,14 +943,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST { Set<Dimension> denormTables = new HashSet<>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { - Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1); + Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(this, cfact, dimsToQuery, cfacts.size() > 1); denormTables.addAll(factDenormTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factDenormTables); } } } else { - denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false)); + denormTables.addAll(deNormCtx.rewriteDenormctx(this, null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index 40ed387..cb26878 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -87,23 +87,19 @@ public class DenormalizationResolver implements ContextRewriter { public static class DenormalizationContext { // map of column name to all references + @Getter private Map<String, Set<ReferencedQueriedColumn>> referencedCols = new HashMap<>(); // candidate table name to all the references columns it needs + @Getter private Map<String, Set<ReferencedQueriedColumn>> tableToRefCols = new HashMap<>(); - private CubeQueryContext cubeql; - // set of all picked references once all candidate tables are picked private Set<PickedReference> pickedRefs = new HashSet<>(); // index on column name for picked references with map from column name to // pickedrefs private Map<String, Set<PickedReference>> pickedReferences = new HashMap<>(); - DenormalizationContext(CubeQueryContext cubeql) { - this.cubeql = cubeql; - } - void addReferencedCol(String col, ReferencedQueriedColumn refer) { Set<ReferencedQueriedColumn> refCols = referencedCols.get(col); if (refCols == null) { @@ -116,7 +112,7 @@ public class DenormalizationResolver implements ContextRewriter { // When candidate table does not have the field, this method checks // if the field can be reached through reference, // if yes adds the ref usage and returns to true, if not returns false. - boolean addRefUsage(CandidateTable table, String col, String srcTbl) throws LensException { + boolean addRefUsage(CubeQueryContext cubeql, CandidateTable table, String col, String srcTbl) throws LensException { // available as referenced col if (referencedCols.containsKey(col)) { for (ReferencedQueriedColumn refer : referencedCols.get(col)) { @@ -144,10 +140,6 @@ public class DenormalizationResolver implements ContextRewriter { return false; } - Map<String, Set<ReferencedQueriedColumn>> getReferencedCols() { - return referencedCols; - } - private void addPickedReference(String col, PickedReference refer) { Set<PickedReference> refCols = pickedReferences.get(col); if (refCols == null) { @@ -169,23 +161,24 @@ public class DenormalizationResolver implements ContextRewriter { return null; } - public Set<Dimension> rewriteDenormctx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, - boolean replaceFact) throws LensException { + public Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, + CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); + log.info("Doing denorm changes for fact :{}", cfact); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact if (cfact != null) { - pickColumnsForTable(cfact.getName()); + pickColumnsForTable(cubeql, cfact.getName()); } // pick referenced columns for dimensions if (dimsToQuery != null && !dimsToQuery.isEmpty()) { for (CandidateDim cdim : dimsToQuery.values()) { - pickColumnsForTable(cdim.getName()); + pickColumnsForTable(cubeql, cdim.getName()); } } // Replace picked reference in all the base trees - replaceReferencedColumns(cfact, replaceFact); + replaceReferencedColumns(cubeql, cfact, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { @@ -195,9 +188,45 @@ public class DenormalizationResolver implements ContextRewriter { } } } + pickedReferences.clear(); + pickedRefs.clear(); return refTbls; } + public boolean hasReferences() { + return !tableToRefCols.isEmpty(); + } + public Set<Dimension> rewriteDenormctxInExpression(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, + CandidateDim> dimsToQuery, ASTNode exprAST) throws LensException { + Set<Dimension> refTbls = new HashSet<>(); + + log.info("Doing denorm changes for expressions in fact :{}", cfact); + if (!tableToRefCols.isEmpty()) { + // pick referenced columns for fact + if (cfact != null) { + pickColumnsForTable(cubeql, cfact.getName()); + } + // pick referenced columns for dimensions + if (dimsToQuery != null && !dimsToQuery.isEmpty()) { + for (CandidateDim cdim : dimsToQuery.values()) { + pickColumnsForTable(cubeql, cdim.getName()); + } + } + // Replace picked reference in expression ast + resolveClause(exprAST); + + // Add the picked references to dimsToQuery + for (PickedReference picked : pickedRefs) { + if (isPickedFor(picked, cfact, dimsToQuery)) { + refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); + cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); + } + } + } + pickedReferences.clear(); + pickedRefs.clear(); + return refTbls; + } // checks if the reference if picked for facts and dimsToQuery passed private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) { if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) { @@ -213,7 +242,7 @@ public class DenormalizationResolver implements ContextRewriter { return false; } - private void pickColumnsForTable(String tbl) throws LensException { + private void pickColumnsForTable(CubeQueryContext cubeql, String tbl) throws LensException { if (tableToRefCols.containsKey(tbl)) { for (ReferencedQueriedColumn refered : tableToRefCols.get(tbl)) { Iterator<ChainRefCol> iter = refered.chainRefCols.iterator(); @@ -237,27 +266,57 @@ public class DenormalizationResolver implements ContextRewriter { } } - private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException { + public void pruneReferences(CubeQueryContext cubeql) { + for (Set<ReferencedQueriedColumn> referencedQueriedColumns : referencedCols.values()) { + for(Iterator<ReferencedQueriedColumn> iterator = referencedQueriedColumns.iterator(); iterator.hasNext();) { + ReferencedQueriedColumn rqc = iterator.next(); + for (Iterator<ChainRefCol> iter = rqc.chainRefCols.iterator(); iter.hasNext();) { + // remove unreachable references + ChainRefCol reference = iter.next(); + if (cubeql.getAutoJoinCtx() == null || !cubeql.getAutoJoinCtx().isReachableDim( + (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) { + log.info("{} is not reachable", reference.getChainName()); + iter.remove(); + } + } + if (rqc.chainRefCols.isEmpty()) { + log.info("The referenced column: {} is not reachable", rqc.col.getName()); + iterator.remove(); + continue; + } + // do column life validation + for (TimeRange range : cubeql.getTimeRanges()) { + if (!rqc.col.isColumnAvailableInTimeRange(range)) { + log.info("The referenced column: {} is not in the range queried", rqc.col.getName()); + iterator.remove(); + break; + } + } + } + } + } + private void replaceReferencedColumns(CubeQueryContext cubeql, CandidateFact cfact, boolean replaceFact) + throws LensException { QueryAST ast = cubeql; boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact .getName()).isEmpty(); if (replaceFact && factRefExists) { ast = cfact; } - resolveClause(cubeql, ast.getSelectAST()); + resolveClause(ast.getSelectAST()); if (factRefExists) { for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - resolveClause(cubeql, storageWhereClauseAST); + resolveClause(storageWhereClauseAST); } } else { - resolveClause(cubeql, ast.getWhereAST()); + resolveClause(ast.getWhereAST()); } - resolveClause(cubeql, ast.getGroupByAST()); - resolveClause(cubeql, ast.getHavingAST()); - resolveClause(cubeql, cubeql.getOrderByAST()); + resolveClause(ast.getGroupByAST()); + resolveClause(ast.getHavingAST()); + resolveClause(cubeql.getOrderByAST()); } - private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException { + private void resolveClause(ASTNode node) throws LensException { if (node == null) { return; } @@ -288,9 +347,22 @@ public class DenormalizationResolver implements ContextRewriter { // recurse down for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); - resolveClause(query, child); + resolveClause(child); + } + } + } + + public Set<String> getNonReachableReferenceFields(String table) { + Set<String> nonReachableFields = new HashSet<>(); + if (tableToRefCols.containsKey(table)) { + for (ReferencedQueriedColumn refcol : tableToRefCols.get(table)) { + if (getReferencedCols().get(refcol.col.getName()).isEmpty()) { + log.info("For table:{}, the column {} is not available", table, refcol.col); + nonReachableFields.add(refcol.col.getName()); + } } } + return nonReachableFields; } } @@ -320,6 +392,14 @@ public class DenormalizationResolver implements ContextRewriter { } } } + private static DenormalizationContext getOrCreateDeNormCtx(TrackDenormContext tdc) { + DenormalizationContext denormCtx = tdc.getDeNormCtx(); + if (denormCtx == null) { + denormCtx = new DenormalizationContext(); + tdc.setDeNormCtx(denormCtx); + } + return denormCtx; + } /** * Find all de-normalized columns, if these columns are not directly available in candidate tables, query will be * replaced with the corresponding table reference @@ -329,33 +409,29 @@ public class DenormalizationResolver implements ContextRewriter { DenormalizationContext denormCtx = cubeql.getDeNormCtx(); if (denormCtx == null) { // Adds all the reference dimensions as eligible for denorm fields - denormCtx = new DenormalizationContext(cubeql); - cubeql.setDeNormCtx(denormCtx); // add ref columns in cube - addRefColsQueried(cubeql, cubeql, denormCtx); + addRefColsQueried(cubeql, cubeql, getOrCreateDeNormCtx(cubeql)); // add ref columns from expressions for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) { for (ExpressionContext ec : ecSet) { for (ExprSpecContext esc : ec.getAllExprs()) { - addRefColsQueried(cubeql, esc, denormCtx); + addRefColsQueried(cubeql, esc, getOrCreateDeNormCtx(esc)); } } } } else if (!denormCtx.tableToRefCols.isEmpty()) { + denormCtx.pruneReferences(cubeql); // In the second iteration of denorm resolver // candidate tables which require denorm fields and the refernces are no // more valid will be pruned if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) { for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) { CandidateFact cfact = i.next(); - if (denormCtx.tableToRefCols.containsKey(cfact.getName())) { - for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cfact.getName())) { - if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) { - log.info("Not considering fact table:{} as column {} is not available", cfact, refcol.col); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(refcol.col.getName())); - i.remove(); - } - } + Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cfact.getName()); + if (!nonReachableFields.isEmpty()) { + log.info("Not considering fact table:{} as columns {} are not available", cfact, nonReachableFields); + cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(nonReachableFields)); + i.remove(); } } if (cubeql.getCandidateFacts().size() == 0) { @@ -368,15 +444,12 @@ public class DenormalizationResolver implements ContextRewriter { for (Dimension dim : cubeql.getDimensions()) { for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) { CandidateDim cdim = i.next(); - if (denormCtx.tableToRefCols.containsKey(cdim.getName())) { - for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cdim.getName())) { - if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) { - log.info("Not considering dim table:{} as column {} is not available", cdim, refcol.col); - cubeql.addDimPruningMsgs(dim, cdim.dimtable, - CandidateTablePruneCause.columnNotFound(refcol.col.getName())); - i.remove(); - } - } + Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cdim.getName()); + if (!nonReachableFields.isEmpty()) { + log.info("Not considering dim table:{} as column {} is not available", cdim, nonReachableFields); + cubeql.addDimPruningMsgs(dim, cdim.dimtable, + CandidateTablePruneCause.columnNotFound(nonReachableFields)); + i.remove(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 60dacdb..0ea0b1c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -154,7 +154,7 @@ class ExpressionResolver implements ContextRewriter { void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws LensException { Set<ExprSpecContext> evalSet = evaluableExpressions.get(cTable); if (evalSet == null) { - evalSet = new LinkedHashSet<ExprSpecContext>(); + evalSet = new LinkedHashSet<>(); evaluableExpressions.put(cTable, evalSet); } // add optional dimensions involved in expressions @@ -196,13 +196,16 @@ class ExpressionResolver implements ContextRewriter { } } - static class ExprSpecContext extends TracksQueriedColumns { + static class ExprSpecContext extends TracksQueriedColumns implements TrackDenormContext { private Set<ExprSpec> exprSpecs = new LinkedHashSet<>(); @Getter @Setter private ASTNode finalAST; @Getter private Set<Dimension> exprDims = new HashSet<>(); + @Getter + @Setter + private DenormalizationResolver.DenormalizationContext deNormCtx; ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException { // replaces table names in expression with aliases in the query @@ -221,6 +224,7 @@ class ExpressionResolver implements ContextRewriter { finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); } + void resolveColumns(CubeQueryContext cubeql) throws LensException { // finds all columns and table aliases in the expression ColumnResolver.getColsForTree(cubeql, finalAST, this, false); @@ -267,13 +271,31 @@ class ExpressionResolver implements ContextRewriter { public String toString() { return HQLParser.getString(finalAST); } + } - @AllArgsConstructor + @RequiredArgsConstructor @ToString private static class PickedExpression { - private String srcAlias; - private ExprSpecContext pickedCtx; + private final String srcAlias; + private final ExprSpecContext pickedCtx; + private transient ASTNode reWrittenAST = null; + + /* + Initialized rewrittenAST as copy of final AST if boolean is passed. Copy would be required if finalAST gets + modified because of denormalization context. + Otherwise, it is final AST reference, without any copy. + */ + void initRewrittenAST(boolean copyFinal) { + if (copyFinal) { + reWrittenAST = MetastoreUtil.copyAST(pickedCtx.getFinalAST()); + } else { + reWrittenAST = pickedCtx.getFinalAST(); + } + } + ASTNode getRewrittenAST() { + return reWrittenAST; + } } static class ExpressionResolverContext { @@ -357,7 +379,7 @@ class ExpressionResolver implements ContextRewriter { boolean isEvaluable = true; for (String col : columns) { if (!cTable.getColumns().contains(col.toLowerCase())) { - if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) { + if (!esc.getDeNormCtx().addRefUsage(cubeql, cTable, col, cTable.getBaseTable().getName())) { // check if it is available as reference, if not expression is not evaluable log.debug("{} = {} is not evaluable in {}", expr, esc, cTable); isEvaluable = false; @@ -379,49 +401,10 @@ class ExpressionResolver implements ContextRewriter { return ec.isEvaluable(cTable); } - /** - * - * @param exprs - * @return - */ - public boolean allNotEvaluable(Set<String> exprs, CandidateTable cTable) { - for (String expr : exprs) { - if (isEvaluable(expr, cTable)) { - return false; - } - } - return true; - } - - public Collection<String> coveringExpressions(Set<String> exprs, CandidateTable cTable) { - Set<String> coveringSet = new HashSet<String>(); - for (String expr : exprs) { - if (isEvaluable(expr, cTable)) { - coveringSet.add(expr); - } - } - return coveringSet; - } - - /** - * Returns true if all passed expressions are evaluable - * - * @param cTable - * @param exprs - * @return - */ - public boolean allEvaluable(CandidateTable cTable, Set<String> exprs) { - for (String expr : exprs) { - if (!isEvaluable(expr, cTable)) { - return false; - } - } - return true; - } - - public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, - QueryAST queryAST) throws LensException { + public Set<Dimension> rewriteExprCtx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension, + CandidateDim> dimsToQuery, QueryAST queryAST) throws LensException { Set<Dimension> exprDims = new HashSet<Dimension>(); + log.info("Picking expressions for fact {} ", cfact); if (!allExprsQueried.isEmpty()) { // pick expressions for fact if (cfact != null) { @@ -433,16 +416,21 @@ class ExpressionResolver implements ContextRewriter { pickExpressionsForTable(cdim); } } - // Replace picked expressions in all the base trees - replacePickedExpressions(cfact, queryAST); - log.debug("Picked expressions: {}", pickedExpressions); + log.info("Picked expressions: {}", pickedExpressions); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { exprDims.addAll(pe.pickedCtx.exprDims); + pe.initRewrittenAST(pe.pickedCtx.deNormCtx.hasReferences()); + exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql, cfact, dimsToQuery, + pe.getRewrittenAST())); } } + // Replace picked expressions in all the base trees + replacePickedExpressions(cfact, queryAST); } + pickedExpressions.clear(); + return exprDims; } @@ -488,7 +476,7 @@ class ExpressionResolver implements ContextRewriter { if (pickedExpressions.containsKey(column)) { PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase()); if (expr != null) { - node.setChild(i, replaceAlias(expr.pickedCtx.finalAST, cubeql)); + node.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); } } } @@ -552,6 +540,21 @@ class ExpressionResolver implements ContextRewriter { if (removed) { continue; } + // Remove expressions for which denormalized columns are no more reachable + esc.getDeNormCtx().pruneReferences(cubeql); + for (String table : esc.getDeNormCtx().getTableToRefCols().keySet()) { + Set<String> nonReachableFields = esc.getDeNormCtx().getNonReachableReferenceFields(table); + if (!nonReachableFields.isEmpty()) { + log.info("Removing expression {} as columns {} are not available", esc, nonReachableFields); + iterator.remove(); + removedEsc.add(esc); + removed = true; + break; + } + } + if (removed) { + continue; + } //remove expressions which are not valid in the timerange queried // If an expression is defined as // ex = a + b // from t1 to t2; http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 11eb8f7..34a562d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -116,7 +116,7 @@ class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedC for (String col : queriedDimAttrs) { if (!cfact.getColumns().contains(col.toLowerCase())) { // check if it available as reference - if (!cubeQl.getDeNormCtx().addRefUsage(cfact, col, cubeQl.getCube().getName())) { + if (!cubeQl.getDeNormCtx().addRefUsage(cubeQl, cfact, col, cubeQl.getCube().getName())) { log.info("column {} is not available in fact table:{} ", col, cfact); return false; } http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java index 89b50f5..f18ae36 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java @@ -160,24 +160,6 @@ public class TimeRangeChecker implements ContextRewriter { } } - // Look at referenced columns through denormalization resolver - // and do column life validation - Map<String, Set<DenormalizationResolver.ReferencedQueriedColumn>> refCols = - cubeql.getDeNormCtx().getReferencedCols(); - for (String col : refCols.keySet()) { - Iterator<DenormalizationResolver.ReferencedQueriedColumn> refColIter = refCols.get(col).iterator(); - while (refColIter.hasNext()) { - DenormalizationResolver.ReferencedQueriedColumn refCol = refColIter.next(); - for (TimeRange range : cubeql.getTimeRanges()) { - if (!refCol.col.isColumnAvailableInTimeRange(range)) { - log.debug("The refernced column: {} is not in the range queried", refCol.col.getName()); - refColIter.remove(); - break; - } - } - } - } - // Remove join paths that have columns with invalid life span AutoJoinContext joinContext = cubeql.getAutoJoinCtx(); if (joinContext == null) { http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java new file mode 100644 index 0000000..5592f70 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +interface TrackDenormContext { + + /** + * Get denormalization context + * + * @return DenormalizationContext + */ + DenormalizationResolver.DenormalizationContext getDeNormCtx(); + + /** + * Set denormalization context + * + * @param deNormCtx DenormalizationContext + */ + void setDeNormCtx(DenormalizationResolver.DenormalizationContext deNormCtx); + +} http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 41ea83d..9b29083 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -578,6 +578,8 @@ public class CubeTestSetup { "dim3 refer", "dim3chain", "id", null, null, 0.0)); cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"), "city name", "cubecity", "name", null, null, 0.0)); + cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"), + "state name", "cubestate", "name", null, null, 0.0)); List<ChainRefCol> references = new ArrayList<>(); references.add(new ChainRefCol("timedatechain1", "full_date")); references.add(new ChainRefCol("timehourchain1", "full_hour")); @@ -592,6 +594,8 @@ public class CubeTestSetup { "City1", null, null, null)); cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"), "City2", null, null, null)); + cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"), + "CityState", null, null, null)); Map<String, JoinChain> joinChains = new HashMap<>(); addCubeChains(joinChains, TEST_CUBE_NAME); @@ -653,7 +657,11 @@ public class CubeTestSetup { exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"), "new measure expr", "myfun(newmeasure)")); exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State", - "concat(cubecity.name, \":\", cubestate.name)")); + new ExprSpec("concat(cityname, \":\", statename_cube)", null, null), + new ExprSpec("substr(concatedcitystate, 10)", null, null))); + exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State", + new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK), + new ExprSpec("substr(concatedcitystate, 10)", null, null))); exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State", "concat('CityState:', cubecity.statename)")); exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state", @@ -1957,6 +1965,7 @@ public class CubeTestSetup { factColumns.add(new FieldSchema("countryid", "int", "country id")); factColumns.add(new FieldSchema("dim1", "string", "dim1")); factColumns.add(new FieldSchema("dim2", "int", "dim2")); + factColumns.add(new FieldSchema("concatedCityState", "string", "citystate")); Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java index f2bb485..5d4e87f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java @@ -215,6 +215,21 @@ public class TestExpressionResolver extends TestQueryRewrite { + " group by concat(cubecity.name, \":\", cubestate.name)", null, getWhereForHourly2days("C1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); } + + @Test + public void testExpressionToExcludeJoin() throws Exception { + // expression which results in join + String hqlQuery = + rewrite("select cityAndStateNew, avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'", + conf); + + String expected = + getExpectedQuery(cubeName, "select substr(testcube.concatedcitystate, 10)" + + " avg(testcube.msr1 + testcube.msr2) FROM ", null, null, " and substr(testcube.dim1, 3) != 'XYZ'" + + " group by substr(testcube.concatedcitystate, 10)", null, getWhereForHourly2days("C1_testfact2_raw")); + TestCubeRewriter.compareQueries(hqlQuery, expected); + } + @Test public void testExpressionInWhereWithJoinClausePassed() throws Exception { assertLensExceptionInRewrite("select cityAndState, avgmsr from testCube tc join citydim cd join statedim sd where " http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java index 6430ed1..677d641 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java @@ -545,7 +545,7 @@ public class TestJoinResolver extends TestQueryRewrite { Configuration conf = new Configuration(hconf); conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3, C4"); String failingQuery = "select testDim2.cityname, testDim2.cityStateCapital FROM testDim2 where " + TWO_DAYS_RANGE; - assertLensExceptionInRewrite(failingQuery, conf, LensCubeErrorCode.NO_REF_COL_AVAILABLE); + assertLensExceptionInRewrite(failingQuery, conf, LensCubeErrorCode.NO_DIM_HAS_COLUMN); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/tools/conf/server/logback.xml ---------------------------------------------------------------------- diff --git a/tools/conf/server/logback.xml b/tools/conf/server/logback.xml index 63ab23c..5173321 100644 --- a/tools/conf/server/logback.xml +++ b/tools/conf/server/logback.xml @@ -41,7 +41,7 @@ <maxHistory>30</maxHistory> </rollingPolicy> <encoder> - <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern> + <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern> </encoder> </appender> @@ -107,7 +107,7 @@ <file>${lens.log.dir}/${queryLogId}.log</file> <append>true</append> <layout class="ch.qos.logback.classic.PatternLayout"> - <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern> + <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern> </layout> </appender> </sift>