http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 82113af..1daeea5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,11 +25,8 @@ import java.util.*; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; -import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; -import org.apache.lens.cube.parse.HQLParser.TreeNode; import org.apache.lens.server.api.error.LensException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -44,9 +41,6 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class ExpressionResolver implements ContextRewriter { - public ExpressionResolver(Configuration conf) { - } - static class ExpressionContext { @Getter private final ExprColumn exprCol; @@ -61,7 +55,7 @@ class ExpressionResolver implements ContextRewriter { private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions = new HashMap<>(); private boolean hasMeasures = false; - public boolean hasMeasures() { + boolean hasMeasures() { return hasMeasures; } @@ -151,11 +145,6 @@ class ExpressionResolver implements ContextRewriter { } void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws LensException { - Set<ExprSpecContext> evalSet = evaluableExpressions.get(cTable); - if (evalSet == null) { - evalSet = new LinkedHashSet<ExprSpecContext>(); - evaluableExpressions.put(cTable, evalSet); - } // add optional dimensions involved in expressions for (String table : esc.getTblAliasToColumns().keySet()) { if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) { @@ -164,7 +153,7 @@ class ExpressionResolver implements ContextRewriter { esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table)); } } - evalSet.add(esc); + evaluableExpressions.computeIfAbsent(cTable, k -> new LinkedHashSet<>()).add(esc); } Set<ASTNode> getAllASTNodes() { @@ -185,41 +174,40 @@ class ExpressionResolver implements ContextRewriter { } boolean isEvaluable(CandidateTable cTable) { - if (directlyAvailableIn.contains(cTable)) { - return true; - } - if (evaluableExpressions.get(cTable) == null) { - return false; - } - return !evaluableExpressions.get(cTable).isEmpty(); + return directlyAvailableIn.contains(cTable) + || (evaluableExpressions.get(cTable) != null && !evaluableExpressions.get(cTable).isEmpty()); } } - static class ExprSpecContext extends TracksQueriedColumns { + static class ExprSpecContext extends TracksQueriedColumns implements TrackDenormContext { private Set<ExprSpec> exprSpecs = new LinkedHashSet<>(); @Getter @Setter private ASTNode finalAST; @Getter private Set<Dimension> exprDims = new HashSet<>(); + @Getter + @Setter + private DenormalizationResolver.DenormalizationContext deNormCtx; ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException { // replaces table names in expression with aliases in the query finalAST = replaceAlias(exprSpec.copyASTNode(), cubeql); exprSpecs.add(exprSpec); } - public ExprSpecContext(ExprSpecContext nested, ExprSpec current, ASTNode node, + ExprSpecContext(ExprSpecContext nested, ExprSpec current, ASTNode node, CubeQueryContext cubeql) throws LensException { exprSpecs.addAll(nested.exprSpecs); exprSpecs.add(current); finalAST = replaceAlias(node, cubeql); } - public void replaceAliasInAST(CubeQueryContext cubeql) + void replaceAliasInAST(CubeQueryContext cubeql) throws LensException { AliasReplacer.extractTabAliasForCol(cubeql, this); finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias()); } + void resolveColumns(CubeQueryContext cubeql) throws LensException { // finds all columns and table aliases in the expression ColumnResolver.getColsForTree(cubeql, finalAST, this, false); @@ -251,28 +239,46 @@ class ExpressionResolver implements ContextRewriter { return null; } - public boolean isValidInTimeRange(final TimeRange range) { + boolean isValidInTimeRange(final TimeRange range) { return isValidFrom(range.getFromDate()) && isValidTill(range.getToDate()); } - public boolean isValidFrom(@NonNull final Date date) { - return (getStartTime() == null) ? true : date.equals(getStartTime()) || date.after(getStartTime()); + boolean isValidFrom(@NonNull final Date date) { + return (getStartTime() == null) || (date.equals(getStartTime()) || date.after(getStartTime())); } - public boolean isValidTill(@NonNull final Date date) { - return (getEndTime() == null) ? true : date.equals(getEndTime()) || date.before(getEndTime()); + boolean isValidTill(@NonNull final Date date) { + return (getEndTime() == null) || (date.equals(getEndTime()) || date.before(getEndTime())); } public String toString() { return HQLParser.getString(finalAST); } + } - @AllArgsConstructor + @RequiredArgsConstructor @ToString private static class PickedExpression { - private String srcAlias; - private ExprSpecContext pickedCtx; + private final String srcAlias; + private final ExprSpecContext pickedCtx; + private transient ASTNode reWrittenAST = null; + + /* + Initialized rewrittenAST as copy of final AST if boolean is passed. Copy would be required if finalAST gets + modified because of denormalization context. + Otherwise, it is final AST reference, without any copy. + */ + void initRewrittenAST(boolean copyFinal) { + if (copyFinal) { + reWrittenAST = MetastoreUtil.copyAST(pickedCtx.getFinalAST()); + } else { + reWrittenAST = pickedCtx.getFinalAST(); + } + } + ASTNode getRewrittenAST() { + return reWrittenAST; + } } static class ExpressionResolverContext { @@ -285,13 +291,7 @@ class ExpressionResolver implements ContextRewriter { this.cubeql = cubeql; } void addExpressionQueried(ExpressionContext expr) { - String exprCol = expr.getExprCol().getName().toLowerCase(); - Set<ExpressionContext> ecSet = allExprsQueried.get(exprCol); - if (ecSet == null) { - ecSet = new LinkedHashSet<ExpressionContext>(); - allExprsQueried.put(exprCol, ecSet); - } - ecSet.add(expr); + allExprsQueried.computeIfAbsent(expr.getExprCol().getName().toLowerCase(), k -> new LinkedHashSet<>()).add(expr); } boolean isQueriedExpression(String column) { @@ -318,7 +318,7 @@ class ExpressionResolver implements ContextRewriter { throw new IllegalArgumentException("no expression available for " + expr + " alias:" + alias); } - public boolean hasMeasures(String expr, CubeInterface cube) { + boolean hasMeasures(String expr, CubeInterface cube) { String alias = cubeql.getAliasForTableName(cube.getName()); ExpressionContext ec = getExpressionContext(expr, alias); boolean hasMeasures = false; @@ -337,7 +337,7 @@ class ExpressionResolver implements ContextRewriter { } //updates all expression specs which are evaluable - public void updateEvaluables(String expr, CandidateTable cTable) + void updateEvaluables(String expr, CandidateTable cTable) throws LensException { String alias = cubeql.getAliasForTableName(cTable.getBaseTable().getName()); ExpressionContext ec = getExpressionContext(expr, alias); @@ -356,7 +356,7 @@ class ExpressionResolver implements ContextRewriter { boolean isEvaluable = true; for (String col : columns) { if (!cTable.getColumns().contains(col.toLowerCase())) { - if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) { + if (!esc.getDeNormCtx().addRefUsage(cubeql, cTable, col, cTable.getBaseTable().getName())) { // check if it is available as reference, if not expression is not evaluable log.debug("{} = {} is not evaluable in {}", expr, esc, cTable); isEvaluable = false; @@ -373,54 +373,15 @@ class ExpressionResolver implements ContextRewriter { } // checks if expr is evaluable - public boolean isEvaluable(String expr, CandidateTable cTable) { + boolean isEvaluable(String expr, CandidateTable cTable) { ExpressionContext ec = getExpressionContext(expr, cubeql.getAliasForTableName(cTable.getBaseTable().getName())); return ec.isEvaluable(cTable); } - /** - * - * @param exprs - * @return - */ - public boolean allNotEvaluable(Set<String> exprs, CandidateTable cTable) { - for (String expr : exprs) { - if (isEvaluable(expr, cTable)) { - return false; - } - } - return true; - } - - public Collection<String> coveringExpressions(Set<String> exprs, CandidateTable cTable) { - Set<String> coveringSet = new HashSet<String>(); - for (String expr : exprs) { - if (isEvaluable(expr, cTable)) { - coveringSet.add(expr); - } - } - return coveringSet; - } - - /** - * Returns true if all passed expressions are evaluable - * - * @param cTable - * @param exprs - * @return - */ - public boolean allEvaluable(CandidateTable cTable, Set<String> exprs) { - for (String expr : exprs) { - if (!isEvaluable(expr, cTable)) { - return false; - } - } - return true; - } - - public Set<Dimension> rewriteExprCtx(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, + Set<Dimension> rewriteExprCtx(CubeQueryContext cubeql, StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, QueryAST queryAST) throws LensException { Set<Dimension> exprDims = new HashSet<Dimension>(); + log.info("Picking expressions for candidate {} ", sc); if (!allExprsQueried.isEmpty()) { // pick expressions for fact if (sc != null) { @@ -432,16 +393,21 @@ class ExpressionResolver implements ContextRewriter { pickExpressionsForTable(cdim); } } - // Replace picked expressions in all the base trees - replacePickedExpressions(sc, queryAST); log.debug("Picked expressions: {}", pickedExpressions); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { exprDims.addAll(pe.pickedCtx.exprDims); + pe.initRewrittenAST(pe.pickedCtx.deNormCtx.hasReferences()); + exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql, sc, dimsToQuery, + pe.getRewrittenAST())); } } + // Replace picked expressions in all the base trees + replacePickedExpressions(sc, queryAST); } + pickedExpressions.clear(); + return exprDims; } @@ -466,27 +432,25 @@ class ExpressionResolver implements ContextRewriter { return; } // Traverse the tree and resolve expression columns - HQLParser.bft(node, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) throws LensException { - ASTNode node = visited.getNode(); - int childcount = node.getChildCount(); - for (int i = 0; i < childcount; i++) { - ASTNode current = (ASTNode) node.getChild(i); - if (current.getToken().getType() == DOT) { - // This is for the case where column name is prefixed by table name - // or table alias - // For example 'select fact.id, dim2.id ...' - // Right child is the column name, left child.ident is table name - ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); - ASTNode colIdent = (ASTNode) current.getChild(1); - String column = colIdent.getText().toLowerCase(); - - if (pickedExpressions.containsKey(column)) { - PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase()); - if (expr != null) { - node.setChild(i, replaceAlias(expr.pickedCtx.finalAST, cubeql)); - } + HQLParser.bft(node, visited -> { + ASTNode node1 = visited.getNode(); + int childcount = node1.getChildCount(); + for (int i = 0; i < childcount; i++) { + ASTNode current = (ASTNode) node1.getChild(i); + if (current.getToken().getType() == DOT) { + // This is for the case where column name is prefixed by table name + // or table alias + // For example 'select fact.id, dim2.id ...' + // Right child is the column name, left child.ident is table name + ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); + ASTNode colIdent = (ASTNode) current.getChild(1); + String column = colIdent.getText().toLowerCase(); + + if (pickedExpressions.containsKey(column)) { + assert tabident != null; + PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase()); + if (expr != null) { + node1.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql)); } } } @@ -515,12 +479,8 @@ class ExpressionResolver implements ContextRewriter { log.debug("{} is not directly evaluable in {}", ec, cTable); if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) { // pick first evaluable expression - Set<PickedExpression> peSet = pickedExpressions.get(ecEntry.getKey()); - if (peSet == null) { - peSet = new HashSet<PickedExpression>(); - pickedExpressions.put(ecEntry.getKey(), peSet); - } - peSet.add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); + pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new HashSet<>()) + .add(new PickedExpression(ec.srcAlias, ec.evaluableExpressions.get(cTable).iterator().next())); } } } @@ -549,6 +509,21 @@ class ExpressionResolver implements ContextRewriter { if (removed) { continue; } + // Remove expressions for which denormalized columns are no more reachable + esc.getDeNormCtx().pruneReferences(cubeql); + for (String table : esc.getDeNormCtx().getTableToRefCols().keySet()) { + Set<String> nonReachableFields = esc.getDeNormCtx().getNonReachableReferenceFields(table); + if (!nonReachableFields.isEmpty()) { + log.info("Removing expression {} as columns {} are not available", esc, nonReachableFields); + iterator.remove(); + removedEsc.add(esc); + removed = true; + break; + } + } + if (removed) { + continue; + } //remove expressions which are not valid in the timerange queried // If an expression is defined as // ex = a + b // from t1 to t2; @@ -614,7 +589,7 @@ class ExpressionResolver implements ContextRewriter { for (Map.Entry<String, Set<String>> entry : cubeql.getTblAliasToColumns().entrySet()) { String alias = entry.getKey(); // skip default alias - if (alias == CubeQueryContext.DEFAULT_TABLE) { + if (Objects.equals(alias, CubeQueryContext.DEFAULT_TABLE)) { continue; } AbstractCubeTable tbl = cubeql.getCubeTableForAlias(alias); @@ -703,24 +678,21 @@ class ExpressionResolver implements ContextRewriter { } private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql) throws LensException { - ASTNode finalAST = MetastoreUtil.copyAST(expr); - HQLParser.bft(finalAST, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) { - ASTNode node = visited.getNode(); - ASTNode parent = null; - if (visited.getParent() != null) { - parent = visited.getParent().getNode(); - } - - if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() == DOT)) { - ASTNode current = (ASTNode) node.getChild(0); - if (current.getToken().getType() == Identifier) { - String tableName = current.getToken().getText().toLowerCase(); - String alias = cubeql.getAliasForTableName(tableName); - if (!alias.equalsIgnoreCase(tableName)) { - node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias))); - } + final ASTNode finalAST = MetastoreUtil.copyAST(expr); + HQLParser.bft(finalAST, visited -> { + ASTNode node = visited.getNode(); + ASTNode parent = null; + if (visited.getParent() != null) { + parent = visited.getParent().getNode(); + } + + if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() == DOT)) { + ASTNode current = (ASTNode) node.getChild(0); + if (current.getToken().getType() == Identifier) { + String tableName = current.getToken().getText().toLowerCase(); + String alias = cubeql.getAliasForTableName(tableName); + if (!alias.equalsIgnoreCase(tableName)) { + node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias))); } } } @@ -734,33 +706,30 @@ class ExpressionResolver implements ContextRewriter { return; } // Traverse the tree and resolve expression columns - HQLParser.bft(expr, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) throws LensException { - ASTNode node = visited.getNode(); - int childcount = node.getChildCount(); - for (int i = 0; i < childcount; i++) { - ASTNode current = (ASTNode) node.getChild(i); - if (current.getToken().getType() == TOK_TABLE_OR_COL && (node != null && node.getToken().getType() != DOT)) { - // Take child ident.totext - ASTNode ident = (ASTNode) current.getChild(0); - String column = ident.getText().toLowerCase(); - if (toReplace.equals(column)) { - node.setChild(i, MetastoreUtil.copyAST(columnAST)); - } - } else if (current.getToken().getType() == DOT) { - // This is for the case where column name is prefixed by table name - // or table alias - // For example 'select fact.id, dim2.id ...' - // Right child is the column name, left child.ident is table name - ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); - ASTNode colIdent = (ASTNode) current.getChild(1); - - String column = colIdent.getText().toLowerCase(); - - if (toReplace.equals(column)) { - node.setChild(i, MetastoreUtil.copyAST(columnAST)); - } + HQLParser.bft(expr, visited -> { + ASTNode node = visited.getNode(); + int childcount = node.getChildCount(); + for (int i = 0; i < childcount; i++) { + ASTNode current = (ASTNode) node.getChild(i); + if (current.getToken().getType() == TOK_TABLE_OR_COL && node.getToken().getType() != DOT) { + // Take child ident.totext + ASTNode ident = (ASTNode) current.getChild(0); + String column = ident.getText().toLowerCase(); + if (toReplace.equals(column)) { + node.setChild(i, MetastoreUtil.copyAST(columnAST)); + } + } else if (current.getToken().getType() == DOT) { + // This is for the case where column name is prefixed by table name + // or table alias + // For example 'select fact.id, dim2.id ...' + // Right child is the column name, left child.ident is table name + ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier); + ASTNode colIdent = (ASTNode) current.getChild(1); + + String column = colIdent.getText().toLowerCase(); + + if (toReplace.equals(column)) { + node.setChild(i, MetastoreUtil.copyAST(columnAST)); } } }
http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java index 48af0c9..94f9c7d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java @@ -104,7 +104,6 @@ public class FieldValidator implements ContextRewriter { conflictingFields.addAll(queriedMsrs); throw new FieldsCannotBeQueriedTogetherException(new ConflictingFields(conflictingFields)); } else { - conflictingFields.addAll(queriedMsrs); throw new FieldsCannotBeQueriedTogetherException(new ConflictingFields(conflictingFields)); } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java index 4664cde..4cae6f8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java @@ -182,7 +182,7 @@ class MaxCoveringFactResolver implements ContextRewriter { private final long seconds; private final long milliseconds; - public TimeCovered(long ms) { + TimeCovered(long ms) { milliseconds = ms % (24 * 60 * 60 * 1000); long seconds = ms / (24 * 60 * 60 * 1000); this.seconds = seconds % (24 * 60 * 60); @@ -194,9 +194,8 @@ class MaxCoveringFactResolver implements ContextRewriter { } public String toString() { - return new StringBuilder().append(days).append(" days, ").append(hours).append(" hours, ").append(minutes) - .append(" minutes, ").append(seconds).append(" seconds, ").append(milliseconds).append(" milliseconds.") - .toString(); + return String.valueOf(days) + " days, " + hours + " hours, " + minutes + + " minutes, " + seconds + " seconds, " + milliseconds + " milliseconds."; } } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java index 0c6465a..50ccab5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PruneCauses.java @@ -18,8 +18,13 @@ */ package org.apache.lens.cube.parse; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.stream.Collectors.toMap; + import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -47,10 +52,7 @@ public class PruneCauses<T> extends HashMap<T, List<CandidateTablePruneCause>> { HashMap<String, List<CandidateTablePruneCause>> detailedMessage = Maps.newHashMap(); for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) { String key = StringUtils.join(entry.getValue(), ","); - if (detailedMessage.get(key) == null) { - detailedMessage.put(key, new ArrayList<CandidateTablePruneCause>()); - } - detailedMessage.get(key).add(entry.getKey()); + detailedMessage.computeIfAbsent(key, k -> new ArrayList<>()).add(entry.getKey()); } return detailedMessage; } @@ -69,10 +71,7 @@ public class PruneCauses<T> extends HashMap<T, List<CandidateTablePruneCause>> { HashMap<CandidateTablePruneCause, List<T>> result = new HashMap<CandidateTablePruneCause, List<T>>(); for (T key : keySet()) { for (CandidateTablePruneCause value : get(key)) { - if (result.get(value) == null) { - result.put(value, new ArrayList<T>()); - } - result.get(value).add(key); + result.computeIfAbsent(value, k -> new ArrayList<>()).add(key); } } return result; @@ -93,12 +92,8 @@ public class PruneCauses<T> extends HashMap<T, List<CandidateTablePruneCause>> { } public String getBriefCause() { - CandidateTablePruneCode maxCause = CandidateTablePruneCode.values()[0]; - for (CandidateTablePruneCause cause : getReversed().keySet()) { - if (cause.getCause().compareTo(maxCause) > 0) { - maxCause = cause.getCause(); - } - } + CandidateTablePruneCode maxCause = getReversed().keySet().stream() + .map(CandidateTablePruneCause::getCause).max(Comparator.naturalOrder()).get(); Map<CandidateTablePruneCause, String> maxCauseMap = Maps.newHashMap(); for (Map.Entry<CandidateTablePruneCause, List<T>> entry : getReversed().entrySet()) { if (entry.getKey().getCause().equals(maxCause)) { @@ -119,5 +114,11 @@ public class PruneCauses<T> extends HashMap<T, List<CandidateTablePruneCause>> { public static final class BriefAndDetailedError { private String brief; private HashMap<String, List<CandidateTablePruneCause>> details; + + Map<HashSet<String>, List<CandidateTablePruneCause>> enhanced() { + return getDetails().entrySet().stream().collect(toMap( + o -> newHashSet(o.getKey().split(",")), + Map.Entry::getValue)); + } } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 832b7a4..310a655 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -124,7 +124,7 @@ class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedC for (String col : queriedDimAttrs) { if (!sc.getColumns().contains(col.toLowerCase())) { // check if it available as reference - if (!cubeQl.getDeNormCtx().addRefUsage(sc, col, cubeQl.getCube().getName())) { + if (!cubeQl.getDeNormCtx().addRefUsage(cubeQl, sc, col, cubeQl.getCube().getName())) { log.info("column {} is not available in fact table:{} ", col, sc); return false; } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index fca86ec..25acb01 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -18,21 +18,44 @@ */ package org.apache.lens.cube.parse; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; -import static org.apache.lens.cube.parse.StorageUtil.*; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.timeDimNotSupported; +import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange; +import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd; +import static org.apache.lens.cube.parse.StorageUtil.processCubeColForDataCompleteness; +import static org.apache.lens.cube.parse.StorageUtil.processExpressionsForCompleteness; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.*; - -import org.apache.lens.cube.metadata.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.TreeSet; + +import org.apache.lens.cube.metadata.AbstractCubeTable; +import org.apache.lens.cube.metadata.CubeFactTable; +import org.apache.lens.cube.metadata.CubeInterface; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.DateUtil; +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.cube.metadata.FactPartition; +import org.apache.lens.cube.metadata.MetastoreUtil; +import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.lib.Node; - import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -67,6 +90,9 @@ public class StorageCandidate implements Candidate, CandidateTable { */ @Getter private TreeSet<UpdatePeriod> validUpdatePeriods = new TreeSet<>(); + @Getter + @Setter + Map<String, SkipUpdatePeriodCode> updatePeriodRejectionCause; private Configuration conf = null; /** @@ -86,7 +112,6 @@ public class StorageCandidate implements Candidate, CandidateTable { @Getter @Setter private QueryAST queryAst; - private Map<Dimension, CandidateDim> dimensions; @Getter private Map<TimeRange, String> rangeToWhere = new LinkedHashMap<>(); @Getter @@ -153,17 +178,15 @@ public class StorageCandidate implements Candidate, CandidateTable { public StorageCandidate(StorageCandidate sc) throws LensException { this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql()); // Copy update periods. - for (UpdatePeriod updatePeriod : sc.getValidUpdatePeriods()) { - this.validUpdatePeriods.add(updatePeriod); - } + this.validUpdatePeriods.addAll(sc.getValidUpdatePeriods()); } private void setMissingExpressions(Set<Dimension> queriedDims) throws LensException { setFromString(String.format("%s", getFromTable())); setWhereString(joinWithAnd( - genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( - CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) - ? getPostSelectionWhereClause() : null)); + genWhereClauseWithDimPartitions(whereString, queriedDims), cubeql.getConf().getBoolean( + CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) + ? getPostSelectionWhereClause() : null)); if (cubeql.getHavingAST() != null) { queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } @@ -195,7 +218,7 @@ public class StorageCandidate implements Candidate, CandidateTable { return whereBuf.toString(); } - static void appendWhereClause(StringBuilder filterCondition, String whereClause, boolean hasMore) { + private static void appendWhereClause(StringBuilder filterCondition, String whereClause, boolean hasMore) { // Make sure we add AND only when there are already some conditions in where // clause if (hasMore && !filterCondition.toString().isEmpty() && !StringUtils.isBlank(whereClause)) { @@ -209,11 +232,11 @@ public class StorageCandidate implements Candidate, CandidateTable { } } - protected String getPostSelectionWhereClause() throws LensException { + private String getPostSelectionWhereClause() throws LensException { return null; } - public void setAnswerableMeasurePhraseIndices(int index) { + void setAnswerableMeasurePhraseIndices(int index) { answerableMeasurePhraseIndices.add(index); } @@ -233,20 +256,20 @@ public class StorageCandidate implements Candidate, CandidateTable { /** * Update Orderby children with final alias used in select * - * @param orderby - * @param select + * @param orderby Order by AST + * @param select Select AST */ private void updateOrderByWithFinalAlias(ASTNode orderby, ASTNode select) { if (orderby == null) { return; } - for(Node orderbyNode : orderby.getChildren()) { + for (Node orderbyNode : orderby.getChildren()) { ASTNode orderBychild = (ASTNode) orderbyNode; - for(Node selectNode : select.getChildren()) { + for (Node selectNode : select.getChildren()) { ASTNode selectChild = (ASTNode) selectNode; if (selectChild.getChildCount() == 2) { if (HQLParser.getString((ASTNode) selectChild.getChild(0)) - .equals(HQLParser.getString((ASTNode) orderBychild.getChild(0)))) { + .equals(HQLParser.getString((ASTNode) orderBychild.getChild(0)))) { ASTNode alias = new ASTNode((ASTNode) selectChild.getChild(1)); orderBychild.replaceChildren(0, 0, alias); break; @@ -356,7 +379,7 @@ public class StorageCandidate implements Candidate, CandidateTable { if (!client.isStorageTableCandidateForRange(name, fromDate, toDate)) { cubeql.addStoragePruningMsg(this, - new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); return false; } else if (!client.partColExists(name, partCol)) { log.info("{} does not exist in {}", partCol, name); @@ -370,7 +393,7 @@ public class StorageCandidate implements Candidate, CandidateTable { Date floorToDate = DateUtil.getFloorDate(toDate, interval); int lookAheadNumParts = conf - .getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(interval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); + .getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(interval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1).iterator(); // add partitions from ceilFrom to floorTo @@ -400,12 +423,12 @@ public class StorageCandidate implements Candidate, CandidateTable { // look-ahead // process time are present TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, interval, 1) - .iterator(); + .iterator(); while (processTimeIter.hasNext()) { Date pdt = processTimeIter.next(); Date nextPdt = processTimeIter.peekNext(); FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, interval, null, - partWhereClauseFormat); + partWhereClauseFormat); updatePartitionStorage(processTimePartition); if (processTimePartition.isFound()) { log.debug("Finer parts not required for look-ahead partition :{}", part); @@ -419,15 +442,15 @@ public class StorageCandidate implements Candidate, CandidateTable { // Get partitions for look ahead process time log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); Set<FactPartition> processTimeParts = getPartitions( - TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(processTimePartCol).build(), - newset, true, failOnPartialData, missingPartitions); + TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(processTimePartCol).build(), + newset, true, failOnPartialData, missingPartitions); log.debug("Look ahead partitions: {}", processTimeParts); TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build(); for (FactPartition pPart : processTimeParts) { log.debug("Looking for finer partitions in pPart: {}", pPart); for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, - partWhereClauseFormat); + partWhereClauseFormat); updatePartitionStorage(innerPart); innerPart.setFound(pPart.isFound()); if (innerPart.isFound()) { @@ -468,10 +491,10 @@ public class StorageCandidate implements Candidate, CandidateTable { } } return - getPartitions(fromDate, ceilFromDate, partCol, partitions, updatePeriods, - addNonExistingParts, failOnPartialData, missingPartitions) - && getPartitions(floorToDate, toDate, partCol, partitions, updatePeriods, - addNonExistingParts, failOnPartialData, missingPartitions); + getPartitions(fromDate, ceilFromDate, partCol, partitions, updatePeriods, + addNonExistingParts, failOnPartialData, missingPartitions) + && getPartitions(floorToDate, toDate, partCol, partitions, updatePeriods, + addNonExistingParts, failOnPartialData, missingPartitions); } /** @@ -506,8 +529,8 @@ public class StorageCandidate implements Candidate, CandidateTable { boolean partColNotSupported = rangeParts.isEmpty(); String storageTableName = getName(); - if (storagePruningMsgs.containsKey(storageTableName)) { - List<CandidateTablePruneCause> causes = storagePruningMsgs.get(storageTableName); + if (storagePruningMsgs.containsKey(this)) { + List<CandidateTablePruneCause> causes = storagePruningMsgs.get(this); // Find the PART_COL_DOES_NOT_EXISTS for (CandidateTablePruneCause cause : causes) { if (cause.getCause().equals(CandidateTablePruneCode.PART_COL_DOES_NOT_EXIST)) { @@ -709,11 +732,11 @@ public class StorageCandidate implements Candidate, CandidateTable { return getName(); } - public void addValidUpdatePeriod(UpdatePeriod updatePeriod) { + void addValidUpdatePeriod(UpdatePeriod updatePeriod) { this.validUpdatePeriods.add(updatePeriod); } - public void updateFromString(CubeQueryContext query, Set<Dimension> queryDims, + void updateFromString(CubeQueryContext query, Set<Dimension> queryDims, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { this.dimsToQuery = dimsToQuery; String alias = cubeql.getAliasForTableName(cubeql.getCube().getName()); @@ -744,4 +767,57 @@ public class StorageCandidate implements Candidate, CandidateTable { } return ret; } + + Set<UpdatePeriod> getAllUpdatePeriods() { + return getFact().getUpdatePeriods().get(getStorageName()); + } + // TODO: move them to upper interfaces for complex candidates. Right now it's unused, so keeping it just here + public boolean isTimeRangeCoverable(TimeRange timeRange) { + return isTimeRangeCoverable(timeRange.getFromDate(), timeRange.getToDate(), getValidUpdatePeriods()); + } + + /** + * Is the time range coverable by given update periods. + * Extracts the max update period, then extracts maximum amount of range from the middle that this update + * period can cover. Then recurses on the ramaining ranges on the left and right side of the extracted chunk + * using one less update period. + * //TODO: add tests if the function is useful. Till then it's untested and unverified. + * @param fromDate From date + * @param toDate To date + * @param periods Update periods to check + * @return Whether time range is coverable by provided update periods or not. + */ + private boolean isTimeRangeCoverable(Date fromDate, Date toDate, Set<UpdatePeriod> periods) { + UpdatePeriod interval = CubeFactTable.maxIntervalInRange(fromDate, toDate, periods); + if (fromDate.equals(toDate)) { + return true; + } else if (periods.isEmpty()) { + return false; + } else { + Set<UpdatePeriod> remaining = Sets.difference(periods, Sets.newHashSet(interval)); + return interval != null + && isTimeRangeCoverable(fromDate, DateUtil.getCeilDate(fromDate, interval), remaining) + && isTimeRangeCoverable(DateUtil.getFloorDate(toDate, interval), toDate, remaining); + } + } + + boolean isUpdatePeriodUseful(UpdatePeriod updatePeriod) { + return cubeql.getTimeRanges().stream().anyMatch(timeRange -> isUpdatePeriodUseful(timeRange, updatePeriod)); + } + + /** + * Is the update period useful for this time range. e.g. for a time range of hours and days, monthly + * and yearly update periods are useless. DAILY and HOURLY are useful + * @param timeRange The time range + * @param updatePeriod Update period + * @return Whether it's useless + */ + private boolean isUpdatePeriodUseful(TimeRange timeRange, UpdatePeriod updatePeriod) { + try { + timeRange.truncate(updatePeriod); + return true; + } catch (LensException e) { + return false; + } + } } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 3029589..bc008ae 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -19,6 +19,7 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing; import java.util.*; @@ -46,12 +47,10 @@ class StorageTableResolver implements ContextRewriter { private final UpdatePeriod maxInterval; // TODO union : Remove this. All partitions are stored in the StorageCandidate. private final Map<String, Set<String>> nonExistingPartitions = new HashMap<>(); - CubeMetastoreClient client; + private CubeMetastoreClient client; private PHASE phase; - private float completenessThreshold; - private String completenessPartCol; - public StorageTableResolver(Configuration conf) { + StorageTableResolver(Configuration conf) { this.conf = conf; this.supportedStorages = getSupportedStorages(conf); this.allStoragesSupported = (supportedStorages == null); @@ -64,7 +63,6 @@ class StorageTableResolver implements ContextRewriter { } else { this.maxInterval = null; } - String formatStr = conf.get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); this.phase = PHASE.first(); } @@ -76,7 +74,7 @@ class StorageTableResolver implements ContextRewriter { return null; } - public boolean isStorageSupportedOnDriver(String storage) { + private boolean isStorageSupportedOnDriver(String storage) { return allStoragesSupported || supportedStorages.contains(storage); } @@ -115,7 +113,7 @@ class StorageTableResolver implements ContextRewriter { * Each candidate in the set is a complex candidate. We will evaluate each one to get * all the partitions needed to answer the query. * - * @param cubeql + * @param cubeql cube query context */ private void resolveStoragePartitions(CubeQueryContext cubeql) throws LensException { Iterator<Candidate> candidateIterator = cubeql.getCandidates().iterator(); @@ -125,7 +123,7 @@ class StorageTableResolver implements ContextRewriter { for (TimeRange range : cubeql.getTimeRanges()) { isComplete &= candidate.evaluateCompleteness(range, range, failOnPartialData); } - if (failOnPartialData && !isComplete) { + if (failOnPartialData && !isComplete) { candidateIterator.remove(); log.info("Not considering candidate:{} as its data is not is not complete", candidate); Set<StorageCandidate> scSet = CandidateUtil.getStorageCandidates(candidate); @@ -136,13 +134,19 @@ class StorageTableResolver implements ContextRewriter { cubeql.addStoragePruningMsg(sc, incompletePartitions(sc.getDataCompletenessMap())); } } + } else if (candidate.getParticipatingPartitions().isEmpty() + && candidate instanceof StorageCandidate + && ((StorageCandidate) candidate).getNonExistingPartitions().isEmpty()) { + candidateIterator.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE)); } } } private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throws LensException { - Set<Dimension> allDims = new HashSet<Dimension>(cubeql.getDimensions()); + Set<Dimension> allDims = new HashSet<>(cubeql.getDimensions()); for (Aliased<Dimension> dim : cubeql.getOptionalDimensions()) { allDims.add(dim.getObject()); } @@ -161,7 +165,7 @@ class StorageTableResolver implements ContextRewriter { i.remove(); continue; } - Set<String> storageTables = new HashSet<String>(); + Set<String> storageTables = new HashSet<>(); Map<String, String> whereClauses = new HashMap<String, String>(); boolean foundPart = false; // TODO union : We have to remove all usages of a deprecated class. @@ -237,6 +241,7 @@ class StorageTableResolver implements ContextRewriter { assert (c instanceof StorageCandidate); StorageCandidate sc = (StorageCandidate) c; String storageTable = sc.getStorageName(); + // first check: if the storage is supported on driver if (!isStorageSupportedOnDriver(storageTable)) { log.info("Skipping storage: {} as it is not supported", storageTable); cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE)); @@ -244,9 +249,8 @@ class StorageTableResolver implements ContextRewriter { continue; } String str = conf.get(CubeQueryConfUtil.getValidStorageTablesKey(sc.getFact().getName())); - List<String> validFactStorageTables = StringUtils.isBlank(str) - ? null - : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); + List<String> validFactStorageTables = + StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); storageTable = sc.getName(); // Check if storagetable is in the list of valid storages. if (validFactStorageTables != null && !validFactStorageTables.contains(storageTable)) { @@ -255,70 +259,91 @@ class StorageTableResolver implements ContextRewriter { it.remove(); continue; } - boolean valid = false; - // There could be multiple causes for the same time range. - Set<CandidateTablePruneCause.CandidateTablePruneCode> pruningCauses = new HashSet<>(); - for (TimeRange range : cubeql.getTimeRanges()) { - boolean columnInRange = client - .isStorageTableCandidateForRange(storageTable, range.getFromDate(), range.getToDate()); - if (!columnInRange) { - pruningCauses.add(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); - continue; - } - boolean partitionColumnExists = client.partColExists(storageTable, range.getPartitionColumn()); - valid = partitionColumnExists; - if (!partitionColumnExists) { - String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn()); - TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); - if (fallBackRange == null) { - log.info("No partitions for range:{}. fallback range: {}", range, fallBackRange); - pruningCauses.add(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); - continue; - } - valid = client.partColExists(storageTable, fallBackRange.getPartitionColumn()) - && client.isStorageTableCandidateForRange(storageTable, fallBackRange.getFromDate(), - fallBackRange.getToDate()); - if (!valid) { - pruningCauses.add(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); - } - } - } - if (!valid) { - it.remove(); - for (CandidateTablePruneCode code : pruningCauses) { - cubeql.addStoragePruningMsg(sc, new CandidateTablePruneCause(code)); - } - continue; - } - List<String> validUpdatePeriods = CubeQueryConfUtil .getStringList(conf, CubeQueryConfUtil.getValidUpdatePeriodsKey(sc.getFact().getName(), sc.getStorageName())); boolean isStorageAdded = false; Map<String, SkipUpdatePeriodCode> skipUpdatePeriodCauses = new HashMap<>(); - // Check for update period. + // Populate valid update periods. for (UpdatePeriod updatePeriod : sc.getFact().getUpdatePeriods().get(sc.getStorageName())) { if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) { - log.info("Skipping update period {} for fact {}", updatePeriod, sc.getFact()); - skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_BIGGER); - continue; - } - if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { - log.info("Skipping update period {} for fact {} for storage {}", updatePeriod, sc.getFact(), storageTable); + // if user supplied max interval, all intervals larger than that are useless. + log.info("Skipping update period {} for candidate {} since it's more than max interval supplied({})", + updatePeriod, sc.getName(), maxInterval); + skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.UPDATE_PERIOD_BIGGER_THAN_MAX); + } else if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) { + // if user supplied valid update periods, other update periods are useless + log.info("Skipping update period {} for candidate {} for storage {} since it's invalid", + updatePeriod, sc.getName(), storageTable); skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.INVALID); - continue; + } else if (!sc.isUpdatePeriodUseful(updatePeriod)) { + // if the storage candidate finds this update useful to keep looking at the time ranges queried + skipUpdatePeriodCauses.put(updatePeriod.toString(), + SkipUpdatePeriodCode.QUERY_INTERVAL_SMALLER_THAN_UPDATE_PERIOD); + } else { + isStorageAdded = true; + sc.addValidUpdatePeriod(updatePeriod); } - isStorageAdded = true; - sc.addValidUpdatePeriod(updatePeriod); } + // this is just for documentation/debugging, so we can see why some update periods are skipped. + if (!skipUpdatePeriodCauses.isEmpty()) { + sc.setUpdatePeriodRejectionCause(skipUpdatePeriodCauses); + } + // if no update periods were added in previous section, we skip this storage candidate if (!isStorageAdded) { - cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.updatePeriodsRejected(skipUpdatePeriodCauses)); + if (skipUpdatePeriodCauses.values().stream().allMatch( + SkipUpdatePeriodCode.QUERY_INTERVAL_SMALLER_THAN_UPDATE_PERIOD::equals)) { + // all update periods bigger than query range, it means time range not answerable. + cubeql.addStoragePruningMsg(sc, + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); + } else { // Update periods are rejected for multiple reasons. + cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.updatePeriodsRejected(skipUpdatePeriodCauses)); + } it.remove(); + } else { + Set<CandidateTablePruneCause> allPruningCauses = new HashSet<>(2); + for (TimeRange range : cubeql.getTimeRanges()) { + CandidateTablePruneCause pruningCauseForThisTimeRange = null; + if (!client.isStorageTableCandidateForRange(storageTable, range.getFromDate(), range.getToDate())) { + //This is the prune cause + pruningCauseForThisTimeRange = + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); + } + //Check partition (or fallback) column existence + else if (cubeql.shouldReplaceTimeDimWithPart()) { + if (!client.partColExists(storageTable, range.getPartitionColumn())) { + pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); + TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); + while (fallBackRange != null) { + pruningCauseForThisTimeRange = null; + if (!client.partColExists(storageTable, fallBackRange.getPartitionColumn())) { + pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); + fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql); + } else { + if (!client.isStorageTableCandidateForRange(storageTable, fallBackRange.getFromDate(), + fallBackRange.getToDate())) { + pruningCauseForThisTimeRange = + new CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE); + } + break; + } + } + } + } + + if(pruningCauseForThisTimeRange != null) { + allPruningCauses.add(pruningCauseForThisTimeRange); + } + } + if (!allPruningCauses.isEmpty()) { + it.remove(); + cubeql.addStoragePruningMsg(sc, allPruningCauses.toArray(new CandidateTablePruneCause[0])); + } } } } - void addNonExistingParts(String name, Set<String> nonExistingParts) { + private void addNonExistingParts(String name, Set<String> nonExistingParts) { nonExistingPartitions.put(name, nonExistingParts); } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java index 87f3ac2..f5cd540 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java @@ -166,7 +166,6 @@ public final class StorageUtil { /** * Get fallback range - * TODO union : Add method level comments * * @param range * @param factName http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java deleted file mode 100644 index e37db8b..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static org.apache.hadoop.hive.ql.parse.HiveParser.*; - -import java.util.*; - -import org.apache.lens.cube.error.ColUnAvailableInTimeRange; -import org.apache.lens.cube.error.ColUnAvailableInTimeRangeException; -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.metadata.join.JoinPath; -import org.apache.lens.cube.parse.join.AutoJoinContext; -import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.plan.PlanUtils; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class TimeRangeChecker implements ContextRewriter { - public TimeRangeChecker(Configuration conf) { - - } - @Override - public void rewriteContext(CubeQueryContext cubeql) throws LensException { - if (cubeql.getCube() == null) { - return; - } - doColLifeValidation(cubeql); - } - private void extractTimeRange(CubeQueryContext cubeql) throws LensException { - // get time range - - // Time range should be direct child of where condition - // TOK_WHERE.TOK_FUNCTION.Identifier Or, it should be right hand child of - // AND condition TOK_WHERE.KW_AND.TOK_FUNCTION.Identifier - if (cubeql.getWhereAST() == null || cubeql.getWhereAST().getChildCount() < 1) { - throw new LensException(LensCubeErrorCode.NO_TIMERANGE_FILTER.getLensErrorInfo()); - } - searchTimeRanges(cubeql.getWhereAST(), cubeql, null, 0); - } - - private void searchTimeRanges(ASTNode root, CubeQueryContext cubeql, ASTNode parent, int childIndex) - throws LensException { - if (root == null) { - return; - } else if (root.getToken().getType() == TOK_FUNCTION) { - ASTNode fname = HQLParser.findNodeByPath(root, Identifier); - if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) { - processTimeRangeFunction(cubeql, root, parent, childIndex); - } - } else { - for (int i = 0; i < root.getChildCount(); i++) { - ASTNode child = (ASTNode) root.getChild(i); - searchTimeRanges(child, cubeql, root, i); - } - } - } - - private String getColumnName(ASTNode node) { - String column = null; - if (node.getToken().getType() == DOT) { - ASTNode colIdent = (ASTNode) node.getChild(1); - column = colIdent.getText().toLowerCase(); - } else if (node.getToken().getType() == TOK_TABLE_OR_COL) { - // Take child ident.totext - ASTNode ident = (ASTNode) node.getChild(0); - column = ident.getText().toLowerCase(); - } - return column; - } - - private void processTimeRangeFunction(CubeQueryContext cubeql, ASTNode timenode, ASTNode parent, int childIndex) - throws LensException { - TimeRange.TimeRangeBuilder builder = TimeRange.getBuilder(); - builder.astNode(timenode); - builder.parent(parent); - builder.childIndex(childIndex); - - String timeDimName = getColumnName((ASTNode) timenode.getChild(1)); - - if (!cubeql.getCube().getTimedDimensions().contains(timeDimName)) { - throw new LensException(LensCubeErrorCode.NOT_A_TIMED_DIMENSION.getLensErrorInfo(), timeDimName); - } - // Replace timeDimName with column which is used for partitioning. Assume - // the same column - // is used as a partition column in all storages of the fact - timeDimName = cubeql.getPartitionColumnOfTimeDim(timeDimName); - builder.partitionColumn(timeDimName); - - String fromDateRaw = PlanUtils.stripQuotes(timenode.getChild(2).getText()); - String toDateRaw = null; - if (timenode.getChildCount() > 3) { - ASTNode toDateNode = (ASTNode) timenode.getChild(3); - if (toDateNode != null) { - toDateRaw = PlanUtils.stripQuotes(timenode.getChild(3).getText()); - } - } - long currentTime = cubeql.getConf().getLong(LensConfConstants.QUERY_CURRENT_TIME_IN_MILLIS, 0); - Date now; - if (currentTime != 0) { - now = new Date(currentTime); - } else { - now = new Date(); - } - builder.fromDate(DateUtil.resolveDate(fromDateRaw, now)); - if (StringUtils.isNotBlank(toDateRaw)) { - builder.toDate(DateUtil.resolveDate(toDateRaw, now)); - } else { - builder.toDate(now); - } - - TimeRange range = builder.build(); - range.validate(); - cubeql.getTimeRanges().add(range); - } - - //TODO union: This can be executed before finding CoveringSets but after denormresolver and joinresolver - private void doColLifeValidation(CubeQueryContext cubeql) throws LensException, - ColUnAvailableInTimeRangeException { - Set<String> cubeColumns = cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()); - if (cubeColumns == null || cubeColumns.isEmpty()) { - // Query doesn't have any columns from cube - return; - } - - for (String col : cubeql.getColumnsQueriedForTable(cubeql.getCube().getName())) { - CubeColumn column = cubeql.getCube().getColumnByName(col); - for (TimeRange range : cubeql.getTimeRanges()) { - if (column == null) { - if (!cubeql.getCube().getTimedDimensions().contains(col)) { - throw new LensException(LensCubeErrorCode.NOT_A_CUBE_COLUMN.getLensErrorInfo(), col); - } - continue; - } - if (!column.isColumnAvailableInTimeRange(range)) { - throwException(column); - } - } - } - - // Look at referenced columns through denormalization resolver - // and do column life validation - Map<String, Set<DenormalizationResolver.ReferencedQueriedColumn>> refCols = - cubeql.getDeNormCtx().getReferencedCols(); - for (String col : refCols.keySet()) { - Iterator<DenormalizationResolver.ReferencedQueriedColumn> refColIter = refCols.get(col).iterator(); - while (refColIter.hasNext()) { - DenormalizationResolver.ReferencedQueriedColumn refCol = refColIter.next(); - for (TimeRange range : cubeql.getTimeRanges()) { - if (!refCol.col.isColumnAvailableInTimeRange(range)) { - log.debug("The refernced column: {} is not in the range queried", refCol.col.getName()); - refColIter.remove(); - break; - } - } - } - } - - // Remove join paths that have columns with invalid life span - AutoJoinContext joinContext = cubeql.getAutoJoinCtx(); - if (joinContext == null) { - return; - } - // Get cube columns which are part of join chain - Set<String> joinColumns = joinContext.getAllJoinPathColumnsOfTable((AbstractCubeTable) cubeql.getCube()); - if (joinColumns == null || joinColumns.isEmpty()) { - return; - } - - // Loop over all cube columns part of join paths - for (String col : joinColumns) { - CubeColumn column = cubeql.getCube().getColumnByName(col); - for (TimeRange range : cubeql.getTimeRanges()) { - if (!column.isColumnAvailableInTimeRange(range)) { - log.info("Timerange queried is not in column life for {}, Removing join paths containing the column", column); - // Remove join paths containing this column - Map<Aliased<Dimension>, List<JoinPath>> allPaths = joinContext.getAllPaths(); - - for (Aliased<Dimension> dimension : allPaths.keySet()) { - List<JoinPath> joinPaths = allPaths.get(dimension); - Iterator<JoinPath> joinPathIterator = joinPaths.iterator(); - - while (joinPathIterator.hasNext()) { - JoinPath path = joinPathIterator.next(); - if (path.containsColumnOfTable(col, (AbstractCubeTable) cubeql.getCube())) { - log.info("Removing join path: {} as columns :{} is not available in the range", path, col); - joinPathIterator.remove(); - if (joinPaths.isEmpty()) { - // This dimension doesn't have any paths left - throw new LensException(LensCubeErrorCode.NO_JOIN_PATH.getLensErrorInfo(), - "No valid join path available for dimension " + dimension + " which would satisfy time range " - + range.getFromDate() + "-" + range.getToDate()); - } - } - } // End loop to remove path - - } // End loop for all paths - } - } // End time range loop - } // End column loop - } - - private void throwException(CubeColumn column) throws ColUnAvailableInTimeRangeException { - - final Long availabilityStartTime = (column.getStartTimeMillisSinceEpoch().isPresent()) - ? column.getStartTimeMillisSinceEpoch().get() : null; - - final Long availabilityEndTime = column.getEndTimeMillisSinceEpoch().isPresent() - ? column.getEndTimeMillisSinceEpoch().get() : null; - - ColUnAvailableInTimeRange col = new ColUnAvailableInTimeRange(column.getName(), availabilityStartTime, - availabilityEndTime); - - throw new ColUnAvailableInTimeRangeException(col); - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java new file mode 100644 index 0000000..5592f70 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +interface TrackDenormContext { + + /** + * Get denormalization context + * + * @return DenormalizationContext + */ + DenormalizationResolver.DenormalizationContext getDeNormCtx(); + + /** + * Set denormalization context + * + * @param deNormCtx DenormalizationContext + */ + void setDeNormCtx(DenormalizationResolver.DenormalizationContext deNormCtx); + +} http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index 8c81166..4f1f808 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -50,13 +50,11 @@ public class UnionQueryWriter { private Map<HQLParser.HashableASTNode, ASTNode> innerToOuterHavingASTs = new HashMap<>(); private Map<String, ASTNode> storageCandidateToSelectAstMap = new HashMap<>(); private AliasDecider aliasDecider = new DefaultAliasDecider(); - private Candidate cand; private CubeQueryContext cubeql; Set<StorageCandidate> storageCandidates; public static final String DEFAULT_MEASURE = "0.0"; public UnionQueryWriter(Candidate cand, CubeQueryContext cubeql) { - this.cand = cand; this.cubeql = cubeql; storageCandidates = CandidateUtil.getStorageCandidates(cand); } http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java index 9fecdbc..3c157ee 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java @@ -67,6 +67,9 @@ public class FactPartitionBasedQueryCostCalculator implements QueryCostCalculato } cost += allTableWeights * getNormalizedUpdatePeriodCost(partition.getPeriod(), driver); } + } else { + // increase cost for every dimtable partition + cost += 1.0; } } return cost; http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index e21dc2a..950534c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -28,6 +28,7 @@ import static org.apache.lens.server.api.util.LensUtil.getHashMap; import static org.testng.Assert.*; import java.text.SimpleDateFormat; + import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; @@ -45,7 +46,10 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; -import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.mapred.SequenceFileInputFormat; @@ -965,6 +969,132 @@ public class TestCubeMetastoreClient { assertTrue(client.getAllFacts(altered).isEmpty()); } + @Test(priority = 1) + public void testUpdatePeriodTableDescriptions() throws LensException, HiveException { + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); + String factName = "testFactWithUpdatePeriodTableDescriptions"; + + for (CubeMeasure measure : cubeMeasures) { + factColumns.add(measure.getColumn()); + } + // add one dimension of the cube + factColumns.add(new FieldSchema("zipcode", "int", "zip")); + FieldSchema itPart = new FieldSchema("it", "string", "date part"); + FieldSchema etPart = new FieldSchema("et", "string", "date part"); + String[] partColNames = new String[] { getDatePartitionKey(), itPart.getName(), etPart.getName() }; + + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + Lists.newArrayList(getDatePartition(), itPart, etPart), + Lists.newArrayList(getDatePartitionKey(), itPart.getName(), etPart.getName())); + StorageTableDesc s2 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + Lists.newArrayList(getDatePartition(), itPart, etPart), + Lists.newArrayList(getDatePartitionKey(), itPart.getName(), etPart.getName())); + + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, hourlyAndDaily, c2, hourlyAndDaily); + Map<String, StorageTableDesc> storageTables = getHashMap(HOURLY + "_" + c1, s1, DAILY + "_" + c1, s2, c2, s2); + Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap = getHashMap(c1, + getHashMap(HOURLY, HOURLY + "_" + c1, DAILY, DAILY + "_" + c1), c2, getHashMap(HOURLY, c2, DAILY, c2)); + + CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, + storageUpdatePeriodMap); + client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables, + storageUpdatePeriodMap); + + assertTrue(client.tableExists(factName)); + Table cubeTbl = client.getHiveTable(factName); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); + + // Assert for storage tables + for (String entry : storageTables.keySet()) { + String storageTableName = getFactOrDimtableStorageTableName(factName, entry); + assertTrue(client.tableExists(storageTableName)); + } + + String c1TableNameHourly = getFactOrDimtableStorageTableName(cubeFact.getName(), HOURLY + "_" + c1); + String c2TableNameHourly = getFactOrDimtableStorageTableName(cubeFact.getName(), c2); + + Table c1TableHourly = client.getHiveTable(c1TableNameHourly); + c1TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), + StoreAllPartitionTimeline.class.getCanonicalName()); + c1TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, itPart.getName()), + StoreAllPartitionTimeline.class.getCanonicalName()); + c1TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), + StoreAllPartitionTimeline.class.getCanonicalName()); + client.pushHiveTable(c1TableHourly); + + Table c2TableHourly = client.getHiveTable(c2TableNameHourly); + c2TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), + EndsAndHolesPartitionTimeline.class.getCanonicalName()); + c2TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, itPart.getName()), + EndsAndHolesPartitionTimeline.class.getCanonicalName()); + c2TableHourly.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), + EndsAndHolesPartitionTimeline.class.getCanonicalName()); + client.pushHiveTable(c2TableHourly); + + assertSameTimelines(factName, new String[] { c1, c2 }, HOURLY, partColNames); + + StoreAllPartitionTimeline timelineDtC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache + .get(factName, c1, HOURLY, getDatePartitionKey())); + StoreAllPartitionTimeline timelineItC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache + .get(factName, c1, HOURLY, itPart.getName())); + StoreAllPartitionTimeline timelineEtC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache + .get(factName, c1, HOURLY, etPart.getName())); + EndsAndHolesPartitionTimeline timelineDt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache + .get(factName, c2, HOURLY, getDatePartitionKey())); + EndsAndHolesPartitionTimeline timelineIt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache + .get(factName, c2, HOURLY, itPart.getName())); + EndsAndHolesPartitionTimeline timelineEt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache + .get(factName, c2, HOURLY, etPart.getName())); + + StoreAllPartitionTimeline timelineC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache + .get(factName, c1, HOURLY, getDatePartitionKey())); + + Map<String, Date> timeParts1 = getTimePartitionByOffsets(getDatePartitionKey(), 0, itPart.getName(), 0, + etPart.getName(), 0); + StoragePartitionDesc partSpec1 = new StoragePartitionDesc(cubeFact.getName(), timeParts1, null, HOURLY); + + Map<String, Date> timeParts2 = getTimePartitionByOffsets(getDatePartitionKey(), 0, etPart.getName(), 1); + Map<String, String> nonTimeSpec = getHashMap(itPart.getName(), "default"); + final StoragePartitionDesc partSpec2 = new StoragePartitionDesc(cubeFact.getName(), timeParts2, nonTimeSpec, + HOURLY); + + Map<String, Date> timeParts3 = getTimePartitionByOffsets(getDatePartitionKey(), 0, etPart.getName(), 0); + final StoragePartitionDesc partSpec3 = new StoragePartitionDesc(cubeFact.getName(), timeParts3, nonTimeSpec, + HOURLY); + + client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1, CubeTableType.FACT); + client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2, CubeTableType.FACT); + PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); + PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey()); + + assertEquals(timeline1Temp.getClass(), StoreAllPartitionTimeline.class); + assertEquals(timeline2Temp.getClass(), EndsAndHolesPartitionTimeline.class); + + assertEquals(client.getAllParts(c1TableNameHourly).size(), 3); + assertEquals(client.getAllParts(c2TableNameHourly).size(), 3); + + assertSameTimelines(factName, new String[] { c1, c2 }, HOURLY, partColNames); + + assertTimeline(timelineDt, timelineDtC1, HOURLY, 0, 0); + assertTimeline(timelineEt, timelineEtC1, HOURLY, 0, 1); + assertTimeline(timelineIt, timelineItC1, HOURLY, 0, 0); + + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c2, etPart.getName())); + + assertNoPartitionNamedLatest(c1TableNameHourly, partColNames); + assertNoPartitionNamedLatest(c2TableNameHourly, partColNames); + + client.dropFact(factName, true); + assertFalse(client.tableExists(factName)); + for (String entry : storageTables.keySet()) { + String storageTableName = getFactOrDimtableStorageTableName(factName, entry); + assertFalse(client.tableExists(storageTableName)); + } + } + @Test(priority = 2) public void testAlterDerivedCube() throws Exception { String name = "alter_derived_cube"; @@ -1238,7 +1368,10 @@ public class TestCubeMetastoreClient { s1.setFieldDelim(":"); storageTables.put(c1, s1); storageTables.put(c4, s1); - factTable.addStorage(c4, hourlyAndDaily); + Map<UpdatePeriod, String> updatePeriodStoragePrefix = new HashMap<>(); + updatePeriodStoragePrefix.put(HOURLY, c4); + updatePeriodStoragePrefix.put(DAILY, c4); + factTable.addStorage(c4, hourlyAndDaily, updatePeriodStoragePrefix); client.alterCubeFactTable(factName, factTable, storageTables, new HashMap<String, String>()); CubeFactTable altered2 = client.getCubeFact(factName); assertTrue(client.tableExists(c1TableName)); @@ -1261,7 +1394,12 @@ public class TestCubeMetastoreClient { assertTrue(client.tableExists(c4TableName)); // add storage - client.addStorage(altered2, c3, hourlyAndDaily, s1); + updatePeriodStoragePrefix.clear(); + updatePeriodStoragePrefix.put(HOURLY, c3); + updatePeriodStoragePrefix.put(DAILY, c3); + Map<String, StorageTableDesc> storageTableDescMap = new HashMap<>(); + storageTableDescMap.put(c3, s1); + client.addStorage(altered2, c3, hourlyAndDaily, storageTableDescMap, updatePeriodStoragePrefix); CubeFactTable altered3 = client.getCubeFact(factName); assertTrue(altered3.getStorages().contains("C3")); assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); @@ -1517,14 +1655,16 @@ public class TestCubeMetastoreClient { for (Partition partition : c1Parts) { partition.setLocation("blah"); partition.setBucketCount(random.nextInt()); - client.updatePartition(factName, c1, partition); + client.updatePartition(factName, c1, partition, HOURLY); } assertSamePartitions(client.getAllParts(c1TableName), c1Parts); for (Partition partition : c2Parts) { partition.setLocation("blah"); partition.setBucketCount(random.nextInt()); } - client.updatePartitions(factName, c2, c2Parts); + Map<UpdatePeriod, List<Partition>> partitionMap = new HashMap<>(); + partitionMap.put(HOURLY, c2Parts); + client.updatePartitions(factName, c2, partitionMap); assertSamePartitions(client.getAllParts(c2TableName), c2Parts); assertSameTimelines(factName, storages, HOURLY, partColNames); @@ -1998,7 +2138,6 @@ public class TestCubeMetastoreClient { timePartCols); Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, updates); Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); - CubeFactTable cubeFactWithParts = new CubeFactTable(CUBE_NAME, factNameWithPart, factColumns, updatePeriods); // create cube fact http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 194ab7c..94d4b40 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -585,9 +585,10 @@ public class CubeTestSetup { "dim3 refer", "dim3chain", "id", null, null, 0.0)); cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"), "city name", "cubecity", "name", null, null, 0.0)); - // union join context cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema(prefix + "cityname", "string", prefix + "city name"), - prefix + "city name", "cubeCityJoinUnionCtx", "name", null, null, 0.0)); + prefix + "city name", "cubeCityJoinUnionCtx", "name", null, null, 0.0)); + cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"), + "state name", "cubestate", "name", null, null, 0.0)); List<ChainRefCol> references = new ArrayList<>(); references.add(new ChainRefCol("timedatechain1", "full_date")); references.add(new ChainRefCol("timehourchain1", "full_hour")); @@ -602,6 +603,8 @@ public class CubeTestSetup { "City1", null, null, null)); cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"), "City2", null, null, null)); + cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"), + "CityState", null, null, null)); Map<String, JoinChain> joinChains = new HashMap<>(); addCubeChains(joinChains, TEST_CUBE_NAME); @@ -663,7 +666,11 @@ public class CubeTestSetup { exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"), "new measure expr", "myfun(newmeasure)")); exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State", - "concat(cubecity.name, \":\", cubestate.name)")); + new ExprSpec("concat(cityname, \":\", statename_cube)", null, null), + new ExprSpec("substr(concatedcitystate, 10)", null, null))); + exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State", + new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK), + new ExprSpec("substr(concatedcitystate, 10)", null, null))); exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State", "concat('CityState:', cubecity.statename)")); exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state", @@ -2120,6 +2127,7 @@ public class CubeTestSetup { factColumns.add(new FieldSchema("countryid", "int", "country id")); factColumns.add(new FieldSchema("dim1", "string", "dim1")); factColumns.add(new FieldSchema("dim2", "int", "dim2")); + factColumns.add(new FieldSchema("concatedCityState", "string", "citystate")); Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>(); Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java index 1a5bd0d..44bf512 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information