http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index d8f1ab4..646dbd6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -169,14 +169,14 @@ public class DenormalizationResolver implements ContextRewriter { return null; } - public Set<Dimension> rewriteDenormctx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, + public Set<Dimension> rewriteDenormctx(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact - if (cfact != null) { - pickColumnsForTable(cfact.getName()); + if (sc != null) { + pickColumnsForTable(sc.getName()); } // pick referenced columns for dimensions if (dimsToQuery != null && !dimsToQuery.isEmpty()) { @@ -185,11 +185,11 @@ public class DenormalizationResolver implements ContextRewriter { } } // Replace picked reference in all the base trees - replaceReferencedColumns(cfact, replaceFact); + replaceReferencedColumns(sc, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { - if (isPickedFor(picked, cfact, dimsToQuery)) { + if (isPickedFor(picked, sc, dimsToQuery)) { refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } @@ -199,8 +199,8 @@ public class DenormalizationResolver implements ContextRewriter { } // checks if the reference if picked for facts and dimsToQuery passed - private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) { - if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) { + private boolean isPickedFor(PickedReference picked, StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery) { + if (sc != null && picked.pickedFor.equalsIgnoreCase(sc.getName())) { return true; } if (dimsToQuery != null) { @@ -237,18 +237,16 @@ public class DenormalizationResolver implements ContextRewriter { } } - private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException { + private void replaceReferencedColumns(StorageCandidate sc, boolean replaceFact) throws LensException { QueryAST ast = cubeql; - boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact + boolean factRefExists = sc != null && tableToRefCols.get(sc.getName()) != null && !tableToRefCols.get(sc .getName()).isEmpty(); if (replaceFact && factRefExists) { - ast = cfact; + ast = sc.getQueryAst(); } resolveClause(cubeql, ast.getSelectAST()); if (factRefExists) { - for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - resolveClause(cubeql, storageWhereClauseAST); - } + resolveClause(cubeql, sc.getQueryAst().getWhereAST()); } else { resolveClause(cubeql, ast.getWhereAST()); } @@ -346,30 +344,28 @@ public class DenormalizationResolver implements ContextRewriter { // candidate tables which require denorm fields and the refernces are no // more valid will be pruned if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { - for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) { - Candidate cand = i.next(); + for (Iterator<StorageCandidate> i = + CandidateUtil.getStorageCandidates(cubeql.getCandidates()).iterator(); i.hasNext();) { + StorageCandidate sc = i.next(); //TODO union : is this happening in pahse 1 or 2 ? - //TODO Union : If phase 2, the below code will not work. Move to phase1 in that case - if (cand instanceof StorageCandidate) { - StorageCandidate sc = (StorageCandidate) cand; + //TODO union : If phase 2, the below code will not work. Move to phase1 in that case if (denormCtx.tableToRefCols.containsKey(sc.getFact().getName())) { for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(sc.getFact().getName())) { if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) { log.info("Not considering storage candidate :{} as column {} is not available", sc, refcol.col); cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound(refcol.col.getName())); - i.remove(); + Collection<Candidate> prunedCandidates = CandidateUtil.filterCandidates(cubeql.getCandidates(), sc); + cubeql.addCandidatePruningMsg(prunedCandidates, + new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED)); } } - } - } else { - throw new LensException("Not a storage candidate!!"); } } if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), cubeql.getColumnsQueriedForTable(cubeql.getCube().getName()).toString()); } - cubeql.pruneCandidateFactSet(CandidateTablePruneCode.COLUMN_NOT_FOUND); + } if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) { for (Dimension dim : cubeql.getDimensions()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 1b8c560..0cf4b1c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -419,13 +419,13 @@ class ExpressionResolver implements ContextRewriter { return true; } - public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, + public Set<Dimension> rewriteExprCtx(StorageCandidate sc, Map<Dimension, CandidateDim> dimsToQuery, QueryAST queryAST) throws LensException { Set<Dimension> exprDims = new HashSet<Dimension>(); if (!allExprsQueried.isEmpty()) { // pick expressions for fact - if (cfact != null) { - pickExpressionsForTable(cfact); + if (sc != null) { + pickExpressionsForTable(sc); } // pick expressions for dimensions if (dimsToQuery != null && !dimsToQuery.isEmpty()) { @@ -434,7 +434,7 @@ class ExpressionResolver implements ContextRewriter { } } // Replace picked expressions in all the base trees - replacePickedExpressions(cfact, queryAST); + replacePickedExpressions(sc, queryAST); log.debug("Picked expressions: {}", pickedExpressions); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { @@ -446,13 +446,11 @@ class ExpressionResolver implements ContextRewriter { return exprDims; } - private void replacePickedExpressions(CandidateFact cfact, QueryAST queryAST) + private void replacePickedExpressions(StorageCandidate sc, QueryAST queryAST) throws LensException { replaceAST(cubeql, queryAST.getSelectAST()); - if (cfact != null) { - for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) { - replaceAST(cubeql, storageWhereClauseAST); - } + if (sc != null) { + replaceAST(cubeql, sc.getQueryAst().getWhereAST()); } else { replaceAST(cubeql, queryAST.getWhereAST()); } http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index 216ae52..6ccf3d8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -19,9 +19,12 @@ package org.apache.lens.cube.parse; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; +import static org.apache.lens.cube.parse.ColumnResolver.addColumnsForSelectExpr; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.lens.cube.metadata.AbstractBaseTable; import org.apache.lens.server.api.error.LensException; @@ -42,19 +45,19 @@ import lombok.extern.slf4j.Slf4j; @Slf4j class GroupbyResolver implements ContextRewriter { + private static final String SELECT_ALIAS_PREFIX = "select_expr"; private final boolean selectPromotionEnabled; private final boolean groupbyPromotionEnabled; public GroupbyResolver(Configuration conf) { - selectPromotionEnabled = - conf.getBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, CubeQueryConfUtil.DEFAULT_ENABLE_SELECT_TO_GROUPBY); - groupbyPromotionEnabled = - conf.getBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, - CubeQueryConfUtil.DEFAULT_ENABLE_GROUP_BY_TO_SELECT); + selectPromotionEnabled = conf + .getBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, CubeQueryConfUtil.DEFAULT_ENABLE_SELECT_TO_GROUPBY); + groupbyPromotionEnabled = conf + .getBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, CubeQueryConfUtil.DEFAULT_ENABLE_GROUP_BY_TO_SELECT); } - private void promoteSelect(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, - List<String> groupByExprs) throws LensException { + private void promoteSelect(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, List<String> groupByExprs) + throws LensException { if (!selectPromotionEnabled) { return; } @@ -79,7 +82,7 @@ class GroupbyResolver implements ContextRewriter { groupbyAST.addChild(exprAST); } else { // no group by ast exist, create one - ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY)); + ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY, "TOK_GROUPBY")); newAST.addChild(exprAST); cubeql.setGroupByAST(newAST); } @@ -97,7 +100,6 @@ class GroupbyResolver implements ContextRewriter { return node != null && node.getToken() != null && !hasTableOrColumn(node); } - /* * Check if table or column used in node */ @@ -115,8 +117,7 @@ class GroupbyResolver implements ContextRewriter { return false; } - private void promoteGroupby(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, - List<String> groupByExprs) + private void promoteGroupby(CubeQueryContext cubeql, List<SelectPhraseContext> selectExprs, List<String> groupByExprs) throws LensException { if (!groupbyPromotionEnabled) { return; @@ -131,12 +132,44 @@ class GroupbyResolver implements ContextRewriter { for (String expr : groupByExprs) { if (!contains(selectExprs, expr)) { ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf()); - addChildAtIndex(index, cubeql.getSelectAST(), exprAST); + ASTNode parent = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR, "TOK_SELEXPR")); + parent.addChild(exprAST); + exprAST.setParent(parent); + addChildAtIndex(index, cubeql.getSelectAST(), parent); + updateSelectPhrase(cubeql, index, parent); index++; } } } + private void updateSelectPhrase(CubeQueryContext cubeql, int index, ASTNode selectExpr) { + int exprInd = index; + ASTNode selectExprChild = (ASTNode) selectExpr.getChild(0); + Set<String> cols = new HashSet<>(); + SelectPhraseContext sel = new SelectPhraseContext(selectExpr); + addColumnsForSelectExpr(sel, selectExpr, cubeql.getSelectAST(), cols); + String alias = selectExpr.getChildCount() > 1 ? selectExpr.getChild(1).getText() : null; + String selectAlias; + String selectFinalAlias = null; + if (alias != null) { + selectFinalAlias = alias; + selectAlias = SELECT_ALIAS_PREFIX + exprInd; + } else if (cols.size() == 1 && (selectExprChild.getToken().getType() == TOK_TABLE_OR_COL + || selectExprChild.getToken().getType() == DOT)) { + // select expression is same as the column + selectAlias = cols.iterator().next().toLowerCase(); + } else { + selectAlias = SELECT_ALIAS_PREFIX + exprInd; + selectFinalAlias = HQLParser.getString(selectExprChild); + } + cubeql.addColumnsQueried(sel.getTblAliasToColumns()); + sel.setSelectAlias(selectAlias); + sel.setFinalAlias(!StringUtils.isBlank(selectFinalAlias) ? "`" + selectFinalAlias + "`" : selectAlias); + sel.setActualAlias(alias != null ? alias.toLowerCase() : null); + cubeql.getSelectPhrases().add(exprInd, sel); + //cubeql.addSelectPhrase(sel); + } + private void addChildAtIndex(int index, ASTNode parent, ASTNode child) { // add the last child int count = parent.getChildCount(); @@ -158,7 +191,7 @@ class GroupbyResolver implements ContextRewriter { List<SelectPhraseContext> selectExprs = getSelectNonAggregateNonMeasureExpressions(cubeql); List<String> groupByExprs = new ArrayList<>(); if (cubeql.getGroupByString() != null) { - String[] gby = getGroupbyExpressions(cubeql.getGroupByAST()).toArray(new String[]{}); + String[] gby = getGroupbyExpressions(cubeql.getGroupByAST()).toArray(new String[] {}); for (String g : gby) { groupByExprs.add(g.trim()); } @@ -228,7 +261,7 @@ class GroupbyResolver implements ContextRewriter { // by the time Groupby resolver is looking for aggregate, all columns should be aliased with correct // alias name. if (cubeql.getCubeTableForAlias(alias) instanceof AbstractBaseTable) { - if (((AbstractBaseTable)cubeql.getCubeTableForAlias(alias)).getExpressionByName(colname) != null) { + if (((AbstractBaseTable) cubeql.getCubeTableForAlias(alias)).getExpressionByName(colname) != null) { return cubeql.getExprCtx().getExpressionContext(colname, alias).hasAggregates(); } } http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java index 7781ba6..d89e7b4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java @@ -1,16 +1,11 @@ package org.apache.lens.cube.parse; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.Set; +import java.util.*; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; -import lombok.Getter; - /** * Represents a join of two candidates */ @@ -22,46 +17,33 @@ public class JoinCandidate implements Candidate { private Candidate childCandidate1; private Candidate childCandidate2; private String toStr; - @Getter - private String alias; + private QueryAST queryAST; + private CubeQueryContext cubeql; - public JoinCandidate(Candidate childCandidate1, Candidate childCandidate2, String alias) { + public JoinCandidate(Candidate childCandidate1, Candidate childCandidate2, CubeQueryContext cubeql) { this.childCandidate1 = childCandidate1; this.childCandidate2 = childCandidate2; - this.alias = alias; - } - - private String getJoinCondition() { - return null; - } - - @Override - public String toHQL() { - return null; - } - - @Override - public QueryAST getQueryAst() { - return null; + this.cubeql = cubeql; } @Override public Collection<String> getColumns() { - return null; + Set<String> columns = new HashSet<>(); + columns.addAll(childCandidate1.getColumns()); + columns.addAll(childCandidate2.getColumns()); + return columns; } @Override public Date getStartTime() { return childCandidate1.getStartTime().after(childCandidate2.getStartTime()) - ? childCandidate1.getStartTime() - : childCandidate2.getStartTime(); + ? childCandidate1.getStartTime() : childCandidate2.getStartTime(); } @Override public Date getEndTime() { return childCandidate1.getEndTime().before(childCandidate2.getEndTime()) - ? childCandidate1.getEndTime() - : childCandidate2.getEndTime(); + ? childCandidate1.getEndTime() : childCandidate2.getEndTime(); } @Override @@ -90,19 +72,35 @@ public class JoinCandidate implements Candidate { * @return */ @Override - public boolean evaluateCompleteness(TimeRange timeRange, boolean failOnPartialData) throws LensException { - return this.childCandidate1.evaluateCompleteness(timeRange, failOnPartialData) && this.childCandidate2 - .evaluateCompleteness(timeRange, failOnPartialData); + public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData) + throws LensException { + return this.childCandidate1.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData) + && this.childCandidate2.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData); } + /** + * @return all the partitions from the children + */ @Override public Set<FactPartition> getParticipatingPartitions() { - return null; + Set<FactPartition> factPartitionsSet = new HashSet<>(); + factPartitionsSet.addAll(childCandidate1.getParticipatingPartitions()); + factPartitionsSet.addAll(childCandidate2.getParticipatingPartitions()); + return factPartitionsSet; } @Override public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) { - return childCandidate1.isExpressionEvaluable(expr) || childCandidate1.isExpressionEvaluable(expr); + return childCandidate1.isExpressionEvaluable(expr) || childCandidate2.isExpressionEvaluable(expr); + } + + @Override + public Set<Integer> getAnswerableMeasurePhraseIndices() { + Set<Integer> mesureIndices = new HashSet<>(); + for (Candidate cand : getChildren()) { + mesureIndices.addAll(cand.getAnswerableMeasurePhraseIndices()); + } + return mesureIndices; } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java index a53e994..cb1cd65 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -18,9 +18,11 @@ */ package org.apache.lens.cube.parse; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; @@ -37,35 +39,36 @@ class LeastPartitionResolver implements ContextRewriter { @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { - if (cubeql.getCube() != null && !cubeql.getCandidateFactSets().isEmpty()) { - Map<Set<CandidateFact>, Integer> factPartCount = new HashMap<Set<CandidateFact>, Integer>(); + if (cubeql.getCube() != null && !cubeql.getCandidates().isEmpty()) { + Map<Candidate, Integer> factPartCount = new HashMap<>(); //The number of partitions being calculated is not the actual number of partitions, // they are number of time values now instead of partitions. // This seems fine, as the less number of time values actually represent the rollups on time. And with // MaxCoveringFactResolver facts with less partitions which are not covering the range would be removed. - for (Set<CandidateFact> facts : cubeql.getCandidateFactSets()) { - factPartCount.put(facts, getPartCount(facts)); + for (Candidate candidate : cubeql.getCandidates()) { + factPartCount.put(candidate, getPartCount(candidate)); } double minPartitions = Collections.min(factPartCount.values()); - for (Iterator<Set<CandidateFact>> i = cubeql.getCandidateFactSets().iterator(); i.hasNext();) { - Set<CandidateFact> facts = i.next(); - if (factPartCount.get(facts) > minPartitions) { - log.info("Not considering facts:{} from candidate fact tables as it requires more partitions to be" - + " queried:{} minimum:{}", facts, factPartCount.get(facts), minPartitions); + for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext(); ) { + Candidate candidate = i.next(); + if (factPartCount.get(candidate) > minPartitions) { + log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}", + candidate, factPartCount.get(candidate), minPartitions); i.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS)); } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.MORE_PARTITIONS); } } - private int getPartCount(Set<CandidateFact> set) { + private int getPartCount(Candidate candidate) { int parts = 0; - for (CandidateFact f : set) { - parts += f.getNumQueriedParts(); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) { + parts += sc.getNumQueriedParts(); } return parts; } http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java index 57c9c44..2522d92 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -21,7 +21,6 @@ package org.apache.lens.cube.parse; import java.util.Collections; import java.util.Iterator; import java.util.Map; -import java.util.Set; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.metadata.UpdatePeriod; @@ -31,11 +30,10 @@ import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; import com.google.common.collect.Maps; - import lombok.extern.slf4j.Slf4j; /** - * Prune candidate fact sets so that the facts except the ones that are covering maximum of range are pruned + * Prune candidates except the ones that are covering maximum of range are pruned */ @Slf4j class MaxCoveringFactResolver implements ContextRewriter { @@ -53,7 +51,7 @@ class MaxCoveringFactResolver implements ContextRewriter { // redundant computation. return; } - if (cubeql.getCube() == null || cubeql.getCandidateFactSets().size() <= 1) { + if (cubeql.getCube() == null || cubeql.getCandidates().size() <= 1) { // nothing to prune. return; } @@ -66,15 +64,13 @@ class MaxCoveringFactResolver implements ContextRewriter { private void resolveByTimeCovered(CubeQueryContext cubeql) { // For each part column, which candidate fact sets are covering how much amount. // Later, we'll maximize coverage for each queried part column. - Map<String, Map<Set<CandidateFact>, Long>> partCountsPerPartCol = Maps.newHashMap(); - //TODO union: max covering set will be calculated based on List<Candidate> - //TODO union: Each candidate will provide Set<FactPartion> using {@link Candidate.getParticipatingPartitions} - for (Set<CandidateFact> facts : cubeql.getCandidateFactSets()) { - for (Map.Entry<String, Long> entry : getTimeCoveredForEachPartCol(facts).entrySet()) { + Map<String, Map<Candidate, Long>> partCountsPerPartCol = Maps.newHashMap(); + for (Candidate cand : cubeql.getCandidates()) { + for (Map.Entry<String, Long> entry : getTimeCoveredForEachPartCol(cand).entrySet()) { if (!partCountsPerPartCol.containsKey(entry.getKey())) { - partCountsPerPartCol.put(entry.getKey(), Maps.<Set<CandidateFact>, Long>newHashMap()); + partCountsPerPartCol.put(entry.getKey(), Maps.<Candidate, Long>newHashMap()); } - partCountsPerPartCol.get(entry.getKey()).put(facts, entry.getValue()); + partCountsPerPartCol.get(entry.getKey()).put(cand, entry.getValue()); } } // for each queried partition, prune fact sets that are covering less range than max @@ -82,29 +78,32 @@ class MaxCoveringFactResolver implements ContextRewriter { if (partCountsPerPartCol.get(partColQueried) != null) { long maxTimeCovered = Collections.max(partCountsPerPartCol.get(partColQueried).values()); TimeCovered timeCovered = new TimeCovered(maxTimeCovered); - Iterator<Set<CandidateFact>> iter = cubeql.getCandidateFactSets().iterator(); + Iterator<Candidate> iter = cubeql.getCandidates().iterator(); while (iter.hasNext()) { - Set<CandidateFact> facts = iter.next(); - Long timeCoveredLong = partCountsPerPartCol.get(partColQueried).get(facts); + Candidate candidate = iter.next(); + Long timeCoveredLong = partCountsPerPartCol.get(partColQueried).get(candidate); if (timeCoveredLong == null) { timeCoveredLong = 0L; } if (timeCoveredLong < maxTimeCovered) { - log.info("Not considering facts:{} from candidate fact tables as it covers less time than the max" - + " for partition column: {} which is: {}", facts, partColQueried, timeCovered); + log.info("Not considering Candidate:{} from Candidate set as it covers less time than the max" + + " for partition column: {} which is: {}", candidate, partColQueried, timeCovered); iter.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.LESS_DATA)); } } } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.lessData(null)); + // cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.lessData(null)); + } private void resolveByDataCompleteness(CubeQueryContext cubeql) { // From the list of candidate fact sets, we calculate the maxDataCompletenessFactor. float maxDataCompletenessFactor = 0f; - for (Set<CandidateFact> facts : cubeql.getCandidateFactSets()) { - float dataCompletenessFactor = computeDataCompletenessFactor(facts); + for (Candidate cand : cubeql.getCandidates()) { + float dataCompletenessFactor = computeDataCompletenessFactor(cand); if (dataCompletenessFactor > maxDataCompletenessFactor) { maxDataCompletenessFactor = dataCompletenessFactor; } @@ -116,27 +115,26 @@ class MaxCoveringFactResolver implements ContextRewriter { } // We prune those candidate fact set, whose dataCompletenessFactor is less than maxDataCompletenessFactor - //TODO union : This needs to work on List<Candidate> - Iterator<Set<CandidateFact>> iter = cubeql.getCandidateFactSets().iterator(); + Iterator<Candidate> iter = cubeql.getCandidates().iterator(); while (iter.hasNext()) { - Set<CandidateFact> facts = iter.next(); - float dataCompletenessFactor = computeDataCompletenessFactor(facts); + Candidate cand = iter.next(); + float dataCompletenessFactor = computeDataCompletenessFactor(cand); if (dataCompletenessFactor < maxDataCompletenessFactor) { - log.info("Not considering facts:{} from candidate fact tables as the dataCompletenessFactor for this:{} is " - + "less than the max:{}", facts, dataCompletenessFactor, maxDataCompletenessFactor); + log.info("Not considering Candidate :{} from the list as the dataCompletenessFactor for this:{} is " + + "less than the max:{}", cand, dataCompletenessFactor, maxDataCompletenessFactor); iter.remove(); + cubeql.addCandidatePruningMsg(cand, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.INCOMPLETE_PARTITION)); } } - cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.incompletePartitions(null)); } - //TODO union : This needs to work on Candidate - private float computeDataCompletenessFactor(Set<CandidateFact> facts) { + private float computeDataCompletenessFactor(Candidate cand) { float completenessFactor = 0f; int numPartition = 0; - for (CandidateFact fact : facts) { - if (fact.getDataCompletenessMap() != null) { - Map<String, Map<String, Float>> completenessMap = fact.getDataCompletenessMap(); + for (StorageCandidate sc : CandidateUtil.getStorageCandidates(cand)) { + if (sc.getDataCompletenessMap() != null) { + Map<String, Map<String, Float>> completenessMap = sc.getDataCompletenessMap(); for (Map<String, Float> partitionCompleteness : completenessMap.values()) { for (Float value : partitionCompleteness.values()) { numPartition++; @@ -145,33 +143,30 @@ class MaxCoveringFactResolver implements ContextRewriter { } } } - return numPartition == 0 ? completenessFactor : completenessFactor/numPartition; + return numPartition == 0 ? completenessFactor : completenessFactor / numPartition; } /** * Returns time covered by fact set for each part column. - * @param facts + * + * @param cand * @return */ - private Map<String, Long> getTimeCoveredForEachPartCol(Set<CandidateFact> facts) { + private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) { Map<String, Long> ret = Maps.newHashMap(); UpdatePeriod smallest = UpdatePeriod.values()[UpdatePeriod.values().length - 1]; - for (CandidateFact fact : facts) { - for (FactPartition part : fact.getPartsQueried()) { - if (part.getPeriod().compareTo(smallest) < 0) { - smallest = part.getPeriod(); - } + for (FactPartition part : cand.getParticipatingPartitions()) { + if (part.getPeriod().compareTo(smallest) < 0) { + smallest = part.getPeriod(); } } PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns(); - for (CandidateFact fact : facts) { - for (FactPartition part : fact.getPartsQueried()) { - if (part.isFound()) { - try { - partitionRangesForPartitionColumns.add(part); - } catch (LensException e) { - log.error("invalid partition: ", e); - } + for (FactPartition part : cand.getParticipatingPartitions()) { + if (part.isFound()) { + try { + partitionRangesForPartitionColumns.add(part); + } catch (LensException e) { + log.error("invalid partition: ", e); } } } @@ -200,17 +195,9 @@ class MaxCoveringFactResolver implements ContextRewriter { } public String toString() { - return new StringBuilder() - .append(days) - .append(" days, ") - .append(hours) - .append(" hours, ") - .append(minutes) - .append(" minutes, ") - .append(seconds) - .append(" seconds, ") - .append(milliseconds) - .append(" milliseconds.").toString(); + return new StringBuilder().append(days).append(" days, ").append(hours).append(" hours, ").append(minutes) + .append(" minutes, ").append(seconds).append(" seconds, ").append(milliseconds).append(" milliseconds.") + .toString(); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java deleted file mode 100644 index 979c24b..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import static org.apache.lens.cube.parse.HQLParser.*; - -import java.util.*; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.server.api.error.LensException; - -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.HiveParser; - -import org.antlr.runtime.CommonToken; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - -/** - * Writes a join query with all the facts involved, with where, groupby and having expressions pushed down to the fact - * queries. - */ -@Slf4j -class MultiFactHQLContext extends SimpleHQLContext { - - private Set<CandidateFact> facts; - private CubeQueryContext query; - private Map<CandidateFact, SimpleHQLContext> factHQLContextMap = new HashMap<>(); - - MultiFactHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery, - Map<CandidateFact, Set<Dimension>> factDimMap, CubeQueryContext query) throws LensException { - super(); - this.query = query; - this.facts = facts; - for (CandidateFact fact : facts) { - if (fact.getStorageTables().size() > 1) { - factHQLContextMap.put(fact, new SingleFactMultiStorageHQLContext(fact, dimsToQuery, query, fact)); - } else { - factHQLContextMap.put(fact, - new SingleFactSingleStorageHQLContext(fact, dimsToQuery, factDimMap.get(fact), query, - DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), fact))); - } - } - } - - protected void setMissingExpressions() throws LensException { - setSelect(getSelectString()); - setFrom(getFromString()); - setWhere(getWhereString()); - setGroupby(getGroupbyString()); - setHaving(getHavingString()); - setOrderby(getOrderbyString()); - } - - private String getOrderbyString() { - return query.getOrderByString(); - } - - private String getHavingString() { - return null; - } - - private String getGroupbyString() { - return null; - } - - private String getWhereString() { - return query.getWhereString(); - } - - public String toHQL() throws LensException { - return query.getInsertClause() + super.toHQL(); - } - - private String getSelectString() throws LensException { - Map<Integer, List<Integer>> selectToFactIndex = new HashMap<>(query.getSelectAST().getChildCount()); - int fi = 1; - for (CandidateFact fact : facts) { - for (int ind : fact.getSelectIndices()) { - if (!selectToFactIndex.containsKey(ind)) { - selectToFactIndex.put(ind, Lists.<Integer>newArrayList()); - } - selectToFactIndex.get(ind).add(fi); - } - fi++; - } - StringBuilder select = new StringBuilder(); - for (int i = 0; i < query.getSelectAST().getChildCount(); i++) { - if (selectToFactIndex.get(i) == null) { - throw new LensException(LensCubeErrorCode.EXPRESSION_NOT_IN_ANY_FACT.getLensErrorInfo(), - HQLParser.getString((ASTNode) query.getSelectAST().getChild(i))); - } - if (selectToFactIndex.get(i).size() == 1) { - select.append("mq").append(selectToFactIndex.get(i).get(0)).append(".") - .append(query.getSelectPhrases().get(i).getSelectAlias()).append(" "); - } else { - select.append("COALESCE("); - String sep = ""; - for (Integer factIndex : selectToFactIndex.get(i)) { - select.append(sep).append("mq").append(factIndex).append(".").append( - query.getSelectPhrases().get(i).getSelectAlias()); - sep = ", "; - } - select.append(") "); - } - select.append(query.getSelectPhrases().get(i).getFinalAlias()); - if (i != query.getSelectAST().getChildCount() - 1) { - select.append(", "); - } - } - return select.toString(); - } - - private String getMultiFactJoinCondition(int i, String dim) { - StringBuilder joinCondition = new StringBuilder(); - if (i <= 1) { - return "".toString(); - } else { - joinCondition.append("mq").append(i - 2).append(".").append(dim).append(" <=> "). - append("mq").append(i - 1).append(".").append(dim); - } - return joinCondition.toString(); - } - - private String getFromString() throws LensException { - StringBuilder fromBuilder = new StringBuilder(); - int aliasCount = 1; - String sep = ""; - for (CandidateFact fact : facts) { - SimpleHQLContext facthql = factHQLContextMap.get(fact); - fromBuilder.append(sep).append("(").append(facthql.toHQL()).append(")").append(" mq").append(aliasCount++); - sep = " full outer join "; - if (!fact.getDimFieldIndices().isEmpty() && aliasCount > 2) { - fromBuilder.append(" on "); - Iterator<Integer> dimIter = fact.getDimFieldIndices().iterator(); - while (dimIter.hasNext()) { - String dim = query.getSelectPhrases().get(dimIter.next()).getSelectAlias(); - fromBuilder.append(getMultiFactJoinCondition(aliasCount, dim)); - if (dimIter.hasNext()) { - fromBuilder.append(" AND "); - } - } - } - } - return fromBuilder.toString(); - } - - - public static ASTNode convertHavingToWhere(ASTNode havingAST, CubeQueryContext context, Set<CandidateFact> cfacts, - AliasDecider aliasDecider) throws LensException { - if (havingAST == null) { - return null; - } - if (isAggregateAST(havingAST) || isTableColumnAST(havingAST) || isNonAggregateFunctionAST(havingAST)) { - // if already present in select, pick alias - String alias = null; - for (CandidateFact fact : cfacts) { - if (fact.isExpressionAnswerable(havingAST, context)) { - alias = fact.addAndGetAliasFromSelect(havingAST, aliasDecider); - return new ASTNode(new CommonToken(HiveParser.Identifier, alias)); - } - } - } - if (havingAST.getChildren() != null) { - for (int i = 0; i < havingAST.getChildCount(); i++) { - ASTNode replaced = convertHavingToWhere((ASTNode) havingAST.getChild(i), context, cfacts, aliasDecider); - havingAST.setChild(i, replaced); - } - } - return havingAST; - } - - public static ASTNode pushDownHaving(ASTNode ast, CubeQueryContext cubeQueryContext, Set<CandidateFact> cfacts) - throws LensException { - if (ast == null) { - return null; - } - if (ast.getType() == HiveParser.KW_AND || ast.getType() == HiveParser.TOK_HAVING) { - List<ASTNode> children = Lists.newArrayList(); - for (Node child : ast.getChildren()) { - ASTNode newChild = pushDownHaving((ASTNode) child, cubeQueryContext, cfacts); - if (newChild != null) { - children.add(newChild); - } - } - if (children.size() == 0) { - return null; - } else if (children.size() == 1) { - return children.get(0); - } else { - ASTNode newASTNode = new ASTNode(ast.getToken()); - for (ASTNode child : children) { - newASTNode.addChild(child); - } - return newASTNode; - } - } - if (isPrimitiveBooleanExpression(ast)) { - CandidateFact fact = pickFactToPushDown(ast, cubeQueryContext, cfacts); - if (fact == null) { - return ast; - } - fact.addToHaving(ast); - return null; - } - return ast; - } - - private static CandidateFact pickFactToPushDown(ASTNode ast, CubeQueryContext cubeQueryContext, Set<CandidateFact> - cfacts) throws LensException { - for (CandidateFact fact : cfacts) { - if (fact.isExpressionAnswerable(ast, cubeQueryContext)) { - return fact; - } - } - return null; - } - -} http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java index 64a9626..b011e47 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java @@ -36,6 +36,8 @@ import lombok.extern.slf4j.Slf4j; @EqualsAndHashCode(callSuper = true) @Slf4j class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedCubeFields { + // position in org.apache.lens.cube.parse.CubeQueryContext.queriedPhrases + private int position; private final ASTNode exprAST; private Boolean aggregate; private String expr; http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java index 62ceb12..77ebe82 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java @@ -18,14 +18,8 @@ */ package org.apache.lens.cube.parse; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.apache.lens.server.api.error.LensException; -import org.apache.commons.lang.StringUtils; - import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -50,7 +44,7 @@ public abstract class SimpleHQLContext implements HQLContextInterface { } SimpleHQLContext(String select, String from, String where, String groupby, String orderby, String having, - Integer limit) { + Integer limit) { this.select = select; this.from = from; this.where = where; @@ -73,6 +67,7 @@ public abstract class SimpleHQLContext implements HQLContextInterface { * <p></p> * Leaving this empty implementation for the case of all expressions being passed in constructor. If other * constructors are used the missing expressions should be set here + * * @throws LensException */ protected void setMissingExpressions() throws LensException { @@ -80,57 +75,6 @@ public abstract class SimpleHQLContext implements HQLContextInterface { public String toHQL() throws LensException { setMissingExpressions(); - String qfmt = getQueryFormat(); - Object[] queryTreeStrings = getQueryTreeStrings(); - if (log.isDebugEnabled()) { - log.debug("qfmt: {} Query strings: {}", qfmt, Arrays.toString(queryTreeStrings)); - } - String baseQuery = String.format(qfmt, queryTreeStrings); - return baseQuery; - } - - private String[] getQueryTreeStrings() throws LensException { - List<String> qstrs = new ArrayList<String>(); - qstrs.add(select); - qstrs.add(from); - if (!StringUtils.isBlank(where)) { - qstrs.add(where); - } - if (!StringUtils.isBlank(groupby)) { - qstrs.add(groupby); - } - if (!StringUtils.isBlank(having)) { - qstrs.add(having); - } - if (!StringUtils.isBlank(orderby)) { - qstrs.add(orderby); - } - if (limit != null) { - qstrs.add(String.valueOf(limit)); - } - return qstrs.toArray(new String[0]); - } - - private final String baseQueryFormat = "SELECT %s FROM %s"; - - private String getQueryFormat() { - StringBuilder queryFormat = new StringBuilder(); - queryFormat.append(baseQueryFormat); - if (!StringUtils.isBlank(where)) { - queryFormat.append(" WHERE %s"); - } - if (!StringUtils.isBlank(groupby)) { - queryFormat.append(" GROUP BY %s"); - } - if (!StringUtils.isBlank(having)) { - queryFormat.append(" HAVING %s"); - } - if (!StringUtils.isBlank(orderby)) { - queryFormat.append(" ORDER BY %s"); - } - if (limit != null) { - queryFormat.append(" LIMIT %s"); - } - return queryFormat.toString(); + return CandidateUtil.buildHQLString(select, from, where, groupby, orderby, having, limit); } } http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java deleted file mode 100644 index 9b48213..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lens.cube.parse; - -import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_ENABLE_STORAGES_UNION; -import static org.apache.lens.cube.parse.CubeQueryConfUtil.ENABLE_STORAGES_UNION; -import static org.apache.lens.cube.parse.HQLParser.*; - -import static org.apache.hadoop.hive.ql.parse.HiveParser.*; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lens.cube.error.LensCubeErrorCode; -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.MetastoreUtil; -import org.apache.lens.server.api.error.LensException; - -import org.apache.hadoop.hive.ql.lib.Node; -import org.apache.hadoop.hive.ql.parse.ASTNode; -import org.apache.hadoop.hive.ql.parse.HiveParser; - -import org.antlr.runtime.CommonToken; - -public class SingleFactMultiStorageHQLContext extends UnionHQLContext { - - private final QueryAST ast; - - private Map<HashableASTNode, ASTNode> innerToOuterASTs = new HashMap<>(); - private AliasDecider aliasDecider = new DefaultAliasDecider(); - - SingleFactMultiStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, - CubeQueryContext query, QueryAST ast) - throws LensException { - super(query, fact); - if (!query.getConf().getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) { - throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo()); - } - this.ast = ast; - processSelectAST(); - processGroupByAST(); - processHavingAST(); - processOrderByAST(); - processLimit(); - setHqlContexts(getUnionContexts(fact, dimsToQuery, query, ast)); - } - - private void processSelectAST() { - ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST()); - ast.setSelectAST(new ASTNode(originalSelectAST.getToken())); - ASTNode outerSelectAST = processSelectExpression(originalSelectAST); - setSelect(getString(outerSelectAST)); - } - - private void processGroupByAST() { - if (ast.getGroupByAST() != null) { - setGroupby(getString(processExpression(ast.getGroupByAST()))); - } - } - - private void processHavingAST() throws LensException { - if (ast.getHavingAST() != null) { - setHaving(getString(processExpression(ast.getHavingAST()))); - ast.setHavingAST(null); - } - } - - - private void processOrderByAST() { - if (ast.getOrderByAST() != null) { - setOrderby(getString(processOrderbyExpression(ast.getOrderByAST()))); - ast.setOrderByAST(null); - } - } - - private void processLimit() { - setLimit(ast.getLimitValue()); - ast.setLimitValue(null); - } - - private ASTNode processExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // iterate over all children of the ast and get outer ast corresponding to it. - for (Node child : astNode.getChildren()) { - outerExpression.addChild(getOuterAST((ASTNode)child)); - } - return outerExpression; - } - - private ASTNode processSelectExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // iterate over all children of the ast and get outer ast corresponding to it. - for (Node node : astNode.getChildren()) { - ASTNode child = (ASTNode)node; - ASTNode outerSelect = new ASTNode(child); - ASTNode selectExprAST = (ASTNode)child.getChild(0); - ASTNode outerAST = getOuterAST(selectExprAST); - outerSelect.addChild(outerAST); - - // has an alias? add it - if (child.getChildCount() > 1) { - outerSelect.addChild(child.getChild(1)); - } - outerExpression.addChild(outerSelect); - } - return outerExpression; - } - - private ASTNode processOrderbyExpression(ASTNode astNode) { - if (astNode == null) { - return null; - } - ASTNode outerExpression = new ASTNode(astNode); - // sample orderby AST looks the following : - /* - TOK_ORDERBY - TOK_TABSORTCOLNAMEDESC - TOK_NULLS_LAST - . - TOK_TABLE_OR_COL - testcube - cityid - TOK_TABSORTCOLNAMEASC - TOK_NULLS_FIRST - . - TOK_TABLE_OR_COL - testcube - stateid - TOK_TABSORTCOLNAMEASC - TOK_NULLS_FIRST - . - TOK_TABLE_OR_COL - testcube - zipcode - */ - for (Node node : astNode.getChildren()) { - ASTNode child = (ASTNode)node; - ASTNode outerOrderby = new ASTNode(child); - ASTNode tokNullsChild = (ASTNode) child.getChild(0); - ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - outerTokNullsChild.addChild(getOuterAST((ASTNode)tokNullsChild.getChild(0))); - outerOrderby.addChild(outerTokNullsChild); - outerExpression.addChild(outerOrderby); - } - return outerExpression; - } - /* - - Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the - inner query - outer query dynamics. The resultant AST is supposed to be used in outer query. - - Base cases: - 1. ast is null => null - 2. ast is aggregate_function(table.column) => add aggregate_function(table.column) to inner select expressions, - generate alias, return aggregate_function(cube.alias). Memoize the mapping - aggregate_function(table.column) => aggregate_function(cube.alias) - Assumption is aggregate_function is transitive i.e. f(a,b,c,d) = f(f(a,b), f(c,d)). SUM, MAX, MIN etc - are transitive, while AVG, COUNT etc are not. For non-transitive aggregate functions, the re-written - query will be incorrect. - 3. ast has aggregates - iterate over children and add the non aggregate nodes as is and recursively get outer ast - for aggregate. - 4. If no aggregates, simply select its alias in outer ast. - 5. If given ast is memorized as mentioned in the above cases, return the mapping. - */ - private ASTNode getOuterAST(ASTNode astNode) { - if (astNode == null) { - return null; - } - if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) { - return innerToOuterASTs.get(new HashableASTNode(astNode)); - } - if (isAggregateAST(astNode)) { - return processAggregate(astNode); - } else if (hasAggregate(astNode)) { - ASTNode outerAST = new ASTNode(astNode); - for (Node child : astNode.getChildren()) { - ASTNode childAST = (ASTNode) child; - if (hasAggregate(childAST)) { - outerAST.addChild(getOuterAST(childAST)); - } else { - outerAST.addChild(childAST); - } - } - return outerAST; - } else { - ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); - ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); - innerSelectExprAST.addChild(innerSelectASTWithoutAlias); - String alias = aliasDecider.decideAlias(astNode); - ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); - innerSelectExprAST.addChild(aliasNode); - addToInnerSelectAST(innerSelectExprAST); - ASTNode outerAST = getDotAST(query.getCube().getName(), alias); - innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); - return outerAST; - } - } - - private ASTNode processAggregate(ASTNode astNode) { - ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode); - ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR)); - innerSelectExprAST.addChild(innerSelectASTWithoutAlias); - String alias = aliasDecider.decideAlias(astNode); - ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias)); - innerSelectExprAST.addChild(aliasNode); - addToInnerSelectAST(innerSelectExprAST); - ASTNode dotAST = getDotAST(query.getCube().getName(), alias); - ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION)); - //TODO: take care or non-transitive aggregate functions - outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText()))); - outerAST.addChild(dotAST); - innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); - return outerAST; - } - - private void addToInnerSelectAST(ASTNode selectExprAST) { - if (ast.getSelectAST() == null) { - ast.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT))); - } - ast.getSelectAST().addChild(selectExprAST); - } - - private static ArrayList<HQLContextInterface> getUnionContexts(CandidateFact fact, Map<Dimension, CandidateDim> - dimsToQuery, CubeQueryContext query, QueryAST ast) - throws LensException { - ArrayList<HQLContextInterface> contexts = new ArrayList<>(); - String alias = query.getAliasForTableName(query.getCube().getName()); - for (String storageTable : fact.getStorageTables()) { - SingleFactSingleStorageHQLContext ctx = new SingleFactSingleStorageHQLContext(fact, storageTable + " " + alias, - dimsToQuery, query, DefaultQueryAST.fromCandidateFact(fact, storageTable, ast)); - contexts.add(ctx); - } - return contexts; - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java deleted file mode 100644 index dbc84ed..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import java.util.Map; -import java.util.Set; - -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.server.api.error.LensException; - -/** - * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being - * queried. - * <p/> - * Updates from string with join clause expanded - */ -class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { - - private final CandidateFact fact; - private String storageAlias; - - SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, - CubeQueryContext query, QueryAST ast) - throws LensException { - this(fact, dimsToQuery, dimsToQuery.keySet(), query, ast); - } - - SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, - Set<Dimension> dimsQueried, CubeQueryContext query, QueryAST ast) - throws LensException { - super(dimsToQuery, dimsQueried, query, ast); - this.fact = fact; - } - - SingleFactSingleStorageHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery, - CubeQueryContext query, QueryAST ast) throws LensException { - this(fact, dimsToQuery, query, ast); - this.storageAlias = storageAlias; - } - - @Override - protected String getFromTable() throws LensException { - if (getQuery().isAutoJoinResolved()) { - if (storageAlias != null) { - return storageAlias; - } else { - return fact.getStorageString(query.getAliasForTableName(query.getCube().getName())); - } - } else { - if (fact.getStorageTables().size() == 1) { - return getQuery().getQBFromString(fact, getDimsToQuery()); - } else { - return storageAlias; - } - } - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/4af769ee/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 22038f3..636b1d0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.parse; +import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; import static org.apache.lens.cube.parse.StorageUtil.*; @@ -31,8 +32,13 @@ import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.util.ReflectionUtils; + +import org.antlr.runtime.CommonToken; import com.google.common.collect.Sets; import lombok.Getter; @@ -45,6 +51,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class StorageCandidate implements Candidate, CandidateTable { + // TODO union : Put comments on member variables. @Getter private final CubeQueryContext cubeql; private final TimeRangeWriter rangeWriter; @@ -57,9 +64,11 @@ public class StorageCandidate implements Candidate, CandidateTable { /** * Valid udpate periods populated by Phase 1. */ + @Getter private TreeSet<UpdatePeriod> validUpdatePeriods = new TreeSet<>(); private Configuration conf = null; - private Map<String, Map<String, Float>> incompleteMeasureData = new HashMap<>(); + @Getter + private Map<String, Map<String, Float>> dataCompletenessMap = new HashMap<>(); private SimpleDateFormat partWhereClauseFormat = null; /** * Participating fact, storage and dimensions for this StorageCandidate @@ -68,10 +77,24 @@ public class StorageCandidate implements Candidate, CandidateTable { private CubeFactTable fact; @Getter private String storageName; + @Getter + @Setter + private QueryAST queryAst; private Map<Dimension, CandidateDim> dimensions; + @Getter private Map<TimeRange, String> rangeToWhere = new LinkedHashMap<>(); @Getter + @Setter + private String whereString; + @Getter + private final Set<Integer> answerableMeasurePhraseIndices = Sets.newHashSet(); + @Getter + @Setter + private String fromString; + @Getter private CubeInterface cube; + @Getter + Map<Dimension, CandidateDim> dimsToQuery; /** * Cached fact columns */ @@ -86,17 +109,17 @@ public class StorageCandidate implements Candidate, CandidateTable { /** * Partition calculated by getPartition() method. */ - private Set<FactPartition> storagePartitions = new HashSet<>(); + @Getter + private Set<FactPartition> participatingPartitions = new HashSet<>(); /** * Non existing partitions */ private Set<String> nonExistingPartitions = new HashSet<>(); @Getter - private String alias = null; + private int numQueriedParts = 0; - public StorageCandidate(CubeInterface cube, CubeFactTable fact, String storageName, String alias, - CubeQueryContext cubeql) { - if ((cube == null) || (fact == null) || (storageName == null) || (alias == null)) { + public StorageCandidate(CubeInterface cube, CubeFactTable fact, String storageName, CubeQueryContext cubeql) { + if ((cube == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.cube = cube; @@ -104,7 +127,6 @@ public class StorageCandidate implements Candidate, CandidateTable { this.cubeql = cubeql; this.storageName = storageName; this.conf = cubeql.getConf(); - this.alias = alias; this.name = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), storageName); rangeWriter = ReflectionUtils.newInstance(conf .getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, @@ -120,19 +142,53 @@ public class StorageCandidate implements Candidate, CandidateTable { .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); } - @Override - public String toHQL() { - return null; + public StorageCandidate(StorageCandidate sc) { + this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql()); + // Copy update periods. + for (UpdatePeriod updatePeriod : sc.getValidUpdatePeriods()) { + this.validUpdatePeriods.add(updatePeriod); + } } - @Override - public QueryAST getQueryAst() { - return null; + static boolean containsAny(Collection<String> srcSet, Collection<String> colSet) { + if (colSet == null || colSet.isEmpty()) { + return true; + } + for (String column : colSet) { + if (srcSet.contains(column)) { + return true; + } + } + return false; + } + + private void setMissingExpressions() throws LensException { + setFromString(String.format("%s", getFromTable())); + setWhereString(joinWithAnd(whereString, null)); + if (cubeql.getHavingAST() != null) { + queryAst.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); + } + } + + public void setAnswerableMeasurePhraseIndices(int index) { + answerableMeasurePhraseIndices.add(index); + } + + public String toHQL() throws LensException { + setMissingExpressions(); + // Check if the picked candidate is a StorageCandidate and in that case + // update the selectAST with final alias. + if (this == cubeql.getPickedCandidate()) { + CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql); + } + return CandidateUtil + .buildHQLString(queryAst.getSelectString(), fromString, whereString, queryAst.getGroupByString(), + queryAst.getOrderByString(), queryAst.getHavingString(), queryAst.getLimitValue()); } @Override public String getStorageString(String alias) { - return null; + return storageName + " " + alias; } @Override @@ -158,6 +214,7 @@ public class StorageCandidate implements Candidate, CandidateTable { @Override public Date getStartTime() { + // TODO union : get storage stat time and take max out of it return fact.getStartTime(); } @@ -211,6 +268,8 @@ public class StorageCandidate implements Candidate, CandidateTable { * * 4.If the monthly partitions are found, check for lookahead partitions and call getPartitions recursively for the * remaining time intervals i.e, [15 sep - 1 oct) and [1 Dec - 15 Dec) + * + * TODO union : Move this into util. */ private boolean getPartitions(Date fromDate, Date toDate, String partCol, Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods, boolean addNonExistingParts, boolean failOnPartialData, @@ -227,25 +286,23 @@ public class StorageCandidate implements Candidate, CandidateTable { if (interval == UpdatePeriod.CONTINUOUS && rangeWriter.getClass().equals(BetweenTimeRangeWriter.class)) { FactPartition part = new FactPartition(partCol, fromDate, interval, null, partWhereClauseFormat); partitions.add(part); - part.getStorageTables().add(name); + part.getStorageTables().add(storageName); part = new FactPartition(partCol, toDate, interval, null, partWhereClauseFormat); partitions.add(part); - part.getStorageTables().add(name); - log.info("Added continuous fact partition for storage table {}", name); + part.getStorageTables().add(storageName); + log.info("Added continuous fact partition for storage table {}", storageName); return true; } if (!client.isStorageTableCandidateForRange(name, fromDate, toDate)) { cubeql.addStoragePruningMsg(this, new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE)); - // skipStorageCauses.put(name, new CandidateTablePruneCause.SkipStorageCause(RANGE_NOT_ANSWERABLE)); return false; } else if (!client.partColExists(name, partCol)) { log.info("{} does not exist in {}", partCol, name); - // skipStorageCauses.put(name, CandidateTablePruneCause.SkipStorageCause.partColDoesNotExist(partCol)); List<String> missingCols = new ArrayList<>(); missingCols.add(partCol); - cubeql.addStoragePruningMsg(this, partitionColumnsMissing(missingCols)); + // cubeql.addStoragePruningMsg(this, partitionColumnsMissing(missingCols)); return false; } @@ -303,7 +360,7 @@ public class StorageCandidate implements Candidate, CandidateTable { log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); Set<FactPartition> processTimeParts = getPartitions( TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(processTimePartCol).build(), - newset, true, false, missingPartitions); + newset, true, failOnPartialData, missingPartitions); log.debug("Look ahead partitions: {}", processTimeParts); TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build(); for (FactPartition pPart : processTimeParts) { @@ -334,12 +391,12 @@ public class StorageCandidate implements Candidate, CandidateTable { // Add non existing partitions for all cases of whether we populate all non existing or not. missingPartitions.add(part); if (!failOnPartialData) { - if (client.isStorageTablePartitionACandidate(name, part.getPartSpec())) { + if (!client.isStorageTablePartitionACandidate(name, part.getPartSpec())) { log.info("Storage tables not eligible"); return false; } partitions.add(part); - part.getStorageTables().add(name); + part.getStorageTables().add(storageName); } } else { log.info("No finer granual partitions exist for {}", part); @@ -367,13 +424,14 @@ public class StorageCandidate implements Candidate, CandidateTable { * 2. getPartitions for timeRange and validUpdatePeriods */ @Override - public boolean evaluateCompleteness(TimeRange timeRange, boolean failOnPartialData) throws LensException { + public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData) + throws LensException { // Check the measure tags. if (!evaluateMeasuresCompleteness(timeRange)) { log - .info("Fact table:{} has partitions with incomplete data: {} for given ranges: {}", fact, incompleteMeasureData, + .info("Fact table:{} has partitions with incomplete data: {} for given ranges: {}", fact, dataCompletenessMap, cubeql.getTimeRanges()); - cubeql.addStoragePruningMsg(this, incompletePartitions(incompleteMeasureData)); + cubeql.addStoragePruningMsg(this, incompletePartitions(dataCompletenessMap)); if (failOnPartialData) { return false; } @@ -387,15 +445,18 @@ public class StorageCandidate implements Candidate, CandidateTable { Set<FactPartition> rangeParts = getPartitions(timeRange, validUpdatePeriods, true, failOnPartialData, missingParts); String partCol = timeRange.getPartitionColumn(); boolean partColNotSupported = rangeParts.isEmpty(); - String storageTableName = getStorageName(); + String storageTableName = getName(); + if (storagePruningMsgs.containsKey(storageTableName)) { List<CandidateTablePruneCause> causes = storagePruningMsgs.get(storageTableName); // Find the PART_COL_DOES_NOT_EXISTS for (CandidateTablePruneCause cause : causes) { if (cause.getCause().equals(CandidateTablePruneCode.PART_COL_DOES_NOT_EXIST)) { - partColNotSupported = cause.getNonExistantPartCols().contains(partCol); + partColNotSupported &= cause.getNonExistantPartCols().contains(partCol); } } + } else { + partColNotSupported = false; } TimeRange prevRange = timeRange; String sep = ""; @@ -421,6 +482,7 @@ public class StorageCandidate implements Candidate, CandidateTable { break; } } + numQueriedParts += rangeParts.size(); if (!unsupportedTimeDims.isEmpty()) { log.info("Not considering fact table:{} as it doesn't support time dimensions: {}", this.getFact(), unsupportedTimeDims); @@ -436,15 +498,15 @@ public class StorageCandidate implements Candidate, CandidateTable { } String extraWhere = extraWhereClauseFallback.toString(); if (!StringUtils.isEmpty(extraWhere)) { - rangeToWhere.put(timeRange, "((" + rangeWriter + rangeToWhere.put(parentTimeRange, "((" + rangeWriter .getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts) + ") and (" + extraWhere + "))"); } else { - rangeToWhere.put(timeRange, rangeWriter + rangeToWhere.put(parentTimeRange, rangeWriter .getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts)); } - // Add all the partitions. storagePartitions contains all the partitions for previous time ranges also. - this.storagePartitions.addAll(rangeParts); + // Add all the partitions. participatingPartitions contains all the partitions for previous time ranges also. + this.participatingPartitions.addAll(rangeParts); return true; } @@ -457,7 +519,7 @@ public class StorageCandidate implements Candidate, CandidateTable { Set<String> measureTag = new HashSet<>(); Map<String, String> tagToMeasureOrExprMap = new HashMap<>(); - processMeasuresFromExprMeasures(cubeql, measureTag, tagToMeasureOrExprMap); + processExpressionsForCompleteness(cubeql, measureTag, tagToMeasureOrExprMap); Set<String> measures = cubeql.getQueriedMsrs(); if (measures == null) { @@ -491,10 +553,10 @@ public class StorageCandidate implements Candidate, CandidateTable { log.info("Completeness for the measure_tag {} is {}, threshold: {}, for the hour {}", tag, completenessResult.getValue(), completenessThreshold, formatter.format(completenessResult.getKey())); String measureorExprFromTag = tagToMeasureOrExprMap.get(tag); - Map<String, Float> incompletePartition = incompleteMeasureData.get(measureorExprFromTag); + Map<String, Float> incompletePartition = dataCompletenessMap.get(measureorExprFromTag); if (incompletePartition == null) { incompletePartition = new HashMap<>(); - incompleteMeasureData.put(measureorExprFromTag, incompletePartition); + dataCompletenessMap.put(measureorExprFromTag, incompletePartition); } incompletePartition.put(formatter.format(completenessResult.getKey()), completenessResult.getValue()); isDataComplete = true; @@ -518,15 +580,49 @@ public class StorageCandidate implements Candidate, CandidateTable { } @Override - public Set<FactPartition> getParticipatingPartitions() { - return null; - } - - @Override public boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr) { return expr.isEvaluable(this); } + /** + * Update selectAST for StorageCandidate + * 1. Delete projected select expression if it's not answerable by StorageCandidate. + * 2. Replace the queried alias with select alias if both are different in a select expr. + * + * @param cubeql + * @throws LensException + */ + + public void updateAnswerableSelectColumns(CubeQueryContext cubeql) throws LensException { + // update select AST with selected fields + int currentChild = 0; + for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) { + ASTNode selectExpr = (ASTNode) queryAst.getSelectAST().getChild(currentChild); + Set<String> exprCols = HQLParser.getColsInExpr(cubeql.getAliasForTableName(cubeql.getCube()), selectExpr); + if (getColumns().containsAll(exprCols)) { + ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); + String alias = cubeql.getSelectPhrases().get(i).getSelectAlias(); + if (aliasNode != null) { + String queryAlias = aliasNode.getText(); + if (!queryAlias.equals(alias)) { + // replace the alias node + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); + queryAst.getSelectAST().getChild(currentChild) + .replaceChildren(selectExpr.getChildCount() - 1, selectExpr.getChildCount() - 1, newAliasNode); + } + } else { + // add column alias + ASTNode newAliasNode = new ASTNode(new CommonToken(HiveParser.Identifier, alias)); + queryAst.getSelectAST().getChild(currentChild).addChild(newAliasNode); + } + } else { + queryAst.getSelectAST().deleteChild(currentChild); + currentChild--; + } + currentChild++; + } + } + @Override public boolean equals(Object obj) { if (super.equals(obj)) { @@ -557,4 +653,37 @@ public class StorageCandidate implements Candidate, CandidateTable { public void addValidUpdatePeriod(UpdatePeriod updatePeriod) { this.validUpdatePeriods.add(updatePeriod); } + + public void updateFromString(CubeQueryContext query, Set<Dimension> queryDims, + Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + this.dimsToQuery = dimsToQuery; + String alias = cubeql.getAliasForTableName(cubeql.getCube().getName()); + fromString = getAliasForTable(alias); + if (query.isAutoJoinResolved()) { + fromString = query.getAutoJoinCtx().getFromString(fromString, this, queryDims, dimsToQuery, query, cubeql); + } + } + + private String getFromTable() throws LensException { + if (cubeql.isAutoJoinResolved()) { + return fromString; + } else { + return cubeql.getQBFromString(this, getDimsToQuery()); + } + } + + public String getAliasForTable(String alias) { + String database = SessionState.get().getCurrentDatabase(); + String ret; + if (alias == null || alias.isEmpty()) { + ret = name; + } else { + ret = name + " " + alias; + } + if (StringUtils.isNotBlank(database) && !"default".equalsIgnoreCase(database)) { + ret = database + "." + ret; + } + return ret; + } + }
