LENS-552: Union support across storage tables in multi fact query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/04f5a822 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/04f5a822 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/04f5a822 Branch: refs/heads/LENS-581 Commit: 04f5a8223f652baa5cfbebec7f8f9a2886df5076 Parents: bf1053b Author: Rajat Khandelwal <[email protected]> Authored: Thu Dec 24 12:39:33 2015 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Thu Dec 24 12:39:33 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/AggregateResolver.java | 12 +- .../apache/lens/cube/parse/AliasReplacer.java | 5 +- .../apache/lens/cube/parse/AutoJoinContext.java | 2 +- .../apache/lens/cube/parse/CandidateFact.java | 130 ++++--------------- .../apache/lens/cube/parse/ColumnResolver.java | 2 +- .../lens/cube/parse/CubeQueryContext.java | 69 +++++----- .../lens/cube/parse/CubeQueryRewriter.java | 2 +- .../apache/lens/cube/parse/DefaultQueryAST.java | 74 +++++++++++ .../cube/parse/DenormalizationResolver.java | 18 +-- .../apache/lens/cube/parse/DimHQLContext.java | 6 +- .../lens/cube/parse/DimOnlyHQLContext.java | 15 +-- .../lens/cube/parse/ExpressionResolver.java | 24 ++-- .../apache/lens/cube/parse/FactHQLContext.java | 65 ---------- .../apache/lens/cube/parse/GroupbyResolver.java | 2 - .../org/apache/lens/cube/parse/HQLParser.java | 1 - .../apache/lens/cube/parse/JoinResolver.java | 17 +-- .../lens/cube/parse/MultiFactHQLContext.java | 46 +++---- .../org/apache/lens/cube/parse/QueryAST.java | 86 ++++++++++++ .../lens/cube/parse/SingleFactHQLContext.java | 96 -------------- .../parse/SingleFactMultiStorageHQLContext.java | 52 +++++--- .../SingleFactSingleStorageHQLContext.java | 85 ++++++++++++ .../lens/cube/parse/StorageTableResolver.java | 43 +++--- .../apache/lens/cube/metadata/DateFactory.java | 11 ++ .../apache/lens/cube/parse/CubeTestSetup.java | 2 +- .../lens/cube/parse/TestBaseCubeQueries.java | 24 ++-- .../lens/cube/parse/TestCubeRewriter.java | 53 ++++++-- 26 files changed, 481 insertions(+), 461 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 39bd1cc..fd7036a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -47,8 +47,6 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j class AggregateResolver implements ContextRewriter { - public AggregateResolver(Configuration conf) { - } @Override public void rewriteContext(CubeQueryContext cubeql) throws LensException { @@ -166,7 +164,7 @@ class AggregateResolver implements ContextRewriter { String colname; if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) { - colname = ((ASTNode) node.getChild(0)).getText(); + colname = node.getChild(0).getText(); } else { // node in 'alias.column' format ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); @@ -193,15 +191,9 @@ class AggregateResolver implements ContextRewriter { throw new LensException(LensCubeErrorCode.NO_DEFAULT_AGGREGATE.getLensErrorInfo(), colname); } ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION)); - fnroot.setParent(node.getParent()); - ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn)); - fnIdentNode.setParent(fnroot); fnroot.addChild(fnIdentNode); - - node.setParent(fnroot); fnroot.addChild(node); - return fnroot; } } else { @@ -224,7 +216,7 @@ class AggregateResolver implements ContextRewriter { String colname; if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) { - colname = ((ASTNode) node.getChild(0)).getText(); + colname = node.getChild(0).getText(); } else { // node in 'alias.column' format ASTNode colIdent = (ASTNode) node.getChild(1); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java index 0656049..e629731 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java @@ -81,7 +81,7 @@ class AliasReplacer implements ContextRewriter { replaceAliases(cubeql.getWhereAST(), 0, colToTableAlias); - replaceAliases(cubeql.getJoinTree(), 0, colToTableAlias); + replaceAliases(cubeql.getJoinAST(), 0, colToTableAlias); // Update the aggregate expression set AggregateResolver.updateAggregates(cubeql.getSelectAST(), cubeql); @@ -183,7 +183,6 @@ class AliasReplacer implements ContextRewriter { ASTNode aliasNode = (ASTNode) node.getChild(0); ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias)); aliasNode.setChild(0, newAliasIdent); - newAliasIdent.setParent(aliasNode); } else { // Just a column ref, we need to make it alias.col // '.' will become the parent node @@ -192,9 +191,7 @@ class AliasReplacer implements ContextRewriter { ASTNode tabRefNode = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL")); tabRefNode.addChild(aliasIdentNode); - aliasIdentNode.setParent(tabRefNode); dot.addChild(tabRefNode); - tabRefNode.setParent(dot); ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName)); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java index 9472506..7f13c6c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java @@ -101,7 +101,7 @@ public class AutoJoinContext { } private JoinClause getJoinClause(CandidateFact fact) { - if (fact == null) { + if (fact == null || !factClauses.containsKey(fact)) { return minCostClause; } return factClauses.get(fact); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 2338ba7..c305244 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -37,14 +37,14 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.antlr.runtime.CommonToken; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.Getter; +import lombok.Setter; /** * Holds context of a candidate fact table. */ -public class CandidateFact implements CandidateTable { +public class CandidateFact implements CandidateTable, QueryAST { final CubeFactTable fact; @Getter private Set<String> storageTables; @@ -52,27 +52,32 @@ public class CandidateFact implements CandidateTable { private int numQueriedParts = 0; @Getter private final Set<FactPartition> partsQueried = Sets.newHashSet(); - @Getter - private final Map<TimeRange, String> rangeToWhereClause = Maps.newHashMap(); private CubeInterface baseTable; + @Getter @Setter private ASTNode selectAST; + @Getter @Setter private ASTNode whereAST; - private ASTNode groupbyAST; + @Getter @Setter + private ASTNode groupByAST; + @Getter @Setter private ASTNode havingAST; - private ASTNode joinTree; + @Getter @Setter + private ASTNode joinAST; + @Getter @Setter + private ASTNode orderByAST; + @Getter @Setter + private Integer limitValue; private List<TimeRangeNode> timenodes = Lists.newArrayList(); private final List<Integer> selectIndices = Lists.newArrayList(); private final List<Integer> dimFieldIndices = Lists.newArrayList(); private Collection<String> columns; @Getter - private final Map<String, String> storgeWhereClauseMap = new HashMap<String, String>(); + private final Map<String, String> storgeWhereClauseMap = new HashMap<>(); @Getter - private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap = - new HashMap<TimeRange, Map<String, LinkedHashSet<FactPartition>>>(); + private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap = new HashMap<>(); @Getter - private final Map<TimeRange, Map<String, String>> rangeToStorageWhereMap = - new HashMap<TimeRange, Map<String, String>>(); + private final Map<TimeRange, Map<String, String>> rangeToStorageWhereMap = new HashMap<>(); CandidateFact(CubeFactTable fact, CubeInterface cube) { this.fact = fact; @@ -114,57 +119,25 @@ public class CandidateFact implements CandidateTable { numQueriedParts += incr; } - private void updateTimeRanges(ASTNode root, ASTNode parent, int childIndex) throws LensException { - if (root == null) { - return; - } else if (root.getToken().getType() == TOK_FUNCTION) { - ASTNode fname = HQLParser.findNodeByPath(root, Identifier); - if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) { - timenodes.add(new TimeRangeNode(root, parent, childIndex)); - } - } else { - for (int i = 0; i < root.getChildCount(); i++) { - ASTNode child = (ASTNode) root.getChild(i); - updateTimeRanges(child, root, i); - } - } - } - // copy ASTs from CubeQueryContext public void copyASTs(CubeQueryContext cubeql) throws LensException { - this.selectAST = HQLParser.copyAST(cubeql.getSelectAST()); - this.whereAST = HQLParser.copyAST(cubeql.getWhereAST()); - if (cubeql.getJoinTree() != null) { - this.joinTree = HQLParser.copyAST(cubeql.getJoinTree()); + setSelectAST(HQLParser.copyAST(cubeql.getSelectAST())); + setWhereAST(HQLParser.copyAST(cubeql.getWhereAST())); + if (cubeql.getJoinAST() != null) { + setJoinAST(HQLParser.copyAST(cubeql.getJoinAST())); } if (cubeql.getGroupByAST() != null) { - this.groupbyAST = HQLParser.copyAST(cubeql.getGroupByAST()); + setGroupByAST(HQLParser.copyAST(cubeql.getGroupByAST())); } if (cubeql.getHavingAST() != null) { - this.havingAST = HQLParser.copyAST(cubeql.getHavingAST()); + setHavingAST(HQLParser.copyAST(cubeql.getHavingAST())); } - // copy timeranges - updateTimeRanges(this.whereAST, null, 0); } public String getWhereClause(String storageTable) { return getStorgeWhereClauseMap().get(storageTable); } - public void updateTimeranges(CubeQueryContext cubeql) throws LensException { - // Update WhereAST with range clause - // resolve timerange positions and replace it by corresponding where clause - for (int i = 0; i < cubeql.getTimeRanges().size(); i++) { - TimeRange range = cubeql.getTimeRanges().get(i); - String rangeWhere = rangeToWhereClause.get(range); - if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); - rangeAST.setParent(timenodes.get(i).parent); - timenodes.get(i).parent.setChild(timenodes.get(i).childIndex, rangeAST); - } - } - } - /** * Update the ASTs to include only the fields queried from this fact, in all the expressions * @@ -322,54 +295,15 @@ public class CandidateFact implements CandidateTable { return null; } - public String getGroupbyTree() { - if (groupbyAST != null) { - return HQLParser.getString(groupbyAST); + @Override + public String getOrderByTree() { + if (orderByAST != null) { + return HQLParser.getString(orderByAST); } return null; } - /** - * @return the selectAST - */ - public ASTNode getSelectAST() { - return selectAST; - } - - /** - * @param selectAST the selectAST to set - */ - public void setSelectAST(ASTNode selectAST) { - this.selectAST = selectAST; - } - - /** - * @return the whereAST - */ - public ASTNode getWhereAST() { - return whereAST; - } - /** - * @param whereAST the whereAST to set - */ - public void setWhereAST(ASTNode whereAST) { - this.whereAST = whereAST; - } - - /** - * @return the havingAST - */ - public ASTNode getHavingAST() { - return havingAST; - } - - /** - * @param havingAST the havingAST to set - */ - public void setHavingAST(ASTNode havingAST) { - this.havingAST = havingAST; - } /** * @return the selectIndices @@ -385,13 +319,9 @@ public class CandidateFact implements CandidateTable { return dimFieldIndices; } - public ASTNode getGroupByAST() { - return groupbyAST; - } - public String getGroupByTree() { - if (groupbyAST != null) { - return HQLParser.getString(groupbyAST); + if (groupByAST != null) { + return HQLParser.getString(groupByAST); } return null; } @@ -413,8 +343,4 @@ public class CandidateFact implements CandidateTable { } return timePartDimensions; } - - public ASTNode getJoinTree() { - return joinTree; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java index b95595a..75aa3f4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java @@ -61,7 +61,7 @@ class ColumnResolver implements ContextRewriter { } getColsForSelectTree(cubeql); getColsForWhereTree(cubeql); - getColsForTree(cubeql, cubeql.getJoinTree(), cubeql); + getColsForTree(cubeql, cubeql.getJoinAST(), cubeql); getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql); getColsForTree(cubeql, cubeql.getHavingAST(), cubeql); getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index 4034a54..1fd1d17 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -21,9 +21,7 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.parse.CubeQueryConfUtil.*; -import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE; +import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import static com.google.common.base.Preconditions.checkArgument; @@ -50,16 +48,11 @@ import org.codehaus.jackson.map.ObjectMapper; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import lombok.*; import lombok.extern.slf4j.Slf4j; @Slf4j -public class CubeQueryContext implements TrackQueriedColumns { +public class CubeQueryContext implements TrackQueriedColumns, QueryAST { public static final String TIME_RANGE_FUNC = "time_range_in"; public static final String NOW = "now"; public static final String DEFAULT_TABLE = "_default_"; @@ -653,7 +646,7 @@ public class CubeQueryContext implements TrackQueriedColumns { return null; } - public ASTNode getJoinTree() { + public ASTNode getJoinAST() { return qb.getParseInfo().getJoinExpr(); } @@ -688,8 +681,8 @@ public class CubeQueryContext implements TrackQueriedColumns { } String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { - String fromString = null; - if (getJoinTree() == null) { + String fromString; + if (getJoinAST() == null) { if (cube != null) { fromString = fact.getStorageString(getAliasForTableName(cube.getName())); } else { @@ -858,6 +851,23 @@ public class CubeQueryContext implements TrackQueriedColumns { @Getter private Collection<CandidateFact> pickedFacts; @Getter private Collection<CandidateDim> pickedDimTables; + private void addRangeClauses(CandidateFact fact) throws LensException { + if (fact != null) { + // resolve timerange positions and replace it by corresponding where clause + for (TimeRange range : getTimeRanges()) { + for (Map.Entry<String, String> entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) { + String table = entry.getKey(); + String rangeWhere = entry.getValue(); + if (!StringUtils.isBlank(rangeWhere)) { + ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); + range.getParent().setChild(range.getChildIndex(), rangeAST); + } + fact.getStorgeWhereClauseMap().put(table, getWhereTree()); + } + } + } + } + public String toHQL() throws LensException { Set<CandidateFact> cfacts = pickCandidateFactToQuery(); Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions); @@ -872,11 +882,13 @@ public class CubeQueryContext implements TrackQueriedColumns { // copy ASTs for each fact for (CandidateFact cfact : cfacts) { cfact.copyASTs(this); - cfact.updateTimeranges(this); - factDimMap.put(cfact, new HashSet<Dimension>(dimsToQuery.keySet())); + factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet())); } - } else { - SingleFactHQLContext.addRangeClauses(this, cfacts.iterator().next()); + } + } + if (cfacts != null) { + for (CandidateFact fact : cfacts) { + addRangeClauses(fact); } } @@ -884,7 +896,7 @@ public class CubeQueryContext implements TrackQueriedColumns { Set<Dimension> exprDimensions = new HashSet<Dimension>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { - Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1); + Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this); exprDimensions.addAll(factExprDimTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factExprDimTables); @@ -892,7 +904,7 @@ public class CubeQueryContext implements TrackQueriedColumns { } } else { // dim only query - exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false)); + exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); @@ -940,25 +952,22 @@ public class CubeQueryContext implements TrackQueriedColumns { } } } - hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap, this); + hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); return hqlContext.toHQL(); } private HQLContextInterface createHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery, - Map<CandidateFact, Set<Dimension>> factDimMap, CubeQueryContext query) throws LensException { + Map<CandidateFact, Set<Dimension>> factDimMap) throws LensException { if (facts == null || facts.size() == 0) { - return new DimOnlyHQLContext(dimsToQuery, query); + return new DimOnlyHQLContext(dimsToQuery, this, this); } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() > 1) { //create single fact with multiple storage context - if (!conf.getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) { - throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo()); - } - return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, query); + return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); } else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() == 1) { // create single fact context - return new SingleFactHQLContext(facts.iterator().next(), dimsToQuery, query); + return new SingleFactSingleStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this); } else { - return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, query); + return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, this); } } @@ -979,10 +988,6 @@ public class CubeQueryContext implements TrackQueriedColumns { return tblAliasToColumns.get(getAliasForTableName(tblName)); } - public void addColumnsQueried(AbstractCubeTable table, String column) { - addColumnsQueried(getAliasForTableName(table.getName()), column); - } - public void addColumnsQueriedWithTimeDimCheck(String alias, String timeDimColumn) { if (!shouldReplaceTimeDimWithPart()) { http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index e0759b0..c1fd0a5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -148,7 +148,7 @@ public class CubeQueryRewriter { // Resolve candidate fact tables and dimension tables for columns queried rewriters.add(candidateTblResolver); // Resolve aggregations and generate base select tree - rewriters.add(new AggregateResolver(conf)); + rewriters.add(new AggregateResolver()); rewriters.add(new GroupbyResolver(conf)); rewriters.add(new FieldValidator()); // Resolve joins and generate base join tree http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java new file mode 100644 index 0000000..0997f37 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import org.apache.lens.server.api.error.LensException; + +import org.apache.hadoop.hive.ql.parse.ASTNode; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class DefaultQueryAST implements QueryAST { + private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST; + private Integer limitValue; + + public String getSelectTree() { + return HQLParser.getString(selectAST); + } + + public String getWhereTree() { + if (whereAST != null) { + return HQLParser.getString(whereAST); + } + return null; + } + + public String getGroupByTree() { + if (groupByAST != null) { + return HQLParser.getString(groupByAST); + } + return null; + } + + + public String getHavingTree() { + if (havingAST != null) { + return HQLParser.getString(havingAST); + } + return null; + } + + @Override + public String getOrderByTree() { + if (orderByAST != null) { + return HQLParser.getString(orderByAST); + } + return null; + } + + public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws + LensException { + return new DefaultQueryAST(ast.getSelectAST(), + HQLParser.parseExpr(fact.getWhereClause(storageTable.substring(storageTable.indexOf(".") + 1))), + ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue()); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java index a576f3a..5c8bd84 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java @@ -281,19 +281,15 @@ public class DenormalizationResolver implements ContextRewriter { } private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException { + QueryAST ast = cubeql; if (replaceFact && (tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact.getName()).isEmpty())) { - resolveClause(cubeql, cfact.getSelectAST()); - resolveClause(cubeql, cfact.getWhereAST()); - resolveClause(cubeql, cfact.getGroupByAST()); - resolveClause(cubeql, cfact.getHavingAST()); - } else { - resolveClause(cubeql, cubeql.getSelectAST()); - resolveClause(cubeql, cubeql.getWhereAST()); - resolveClause(cubeql, cubeql.getGroupByAST()); - resolveClause(cubeql, cubeql.getHavingAST()); - + ast = cfact; } + resolveClause(cubeql, ast.getSelectAST()); + resolveClause(cubeql, ast.getWhereAST()); + resolveClause(cubeql, ast.getGroupByAST()); + resolveClause(cubeql, ast.getHavingAST()); resolveClause(cubeql, cubeql.getOrderByAST()); } @@ -320,11 +316,9 @@ public class DenormalizationResolver implements ContextRewriter { ASTNode newTableNode = new ASTNode(new CommonToken(HiveParser.Identifier, query.getAliasForTableName(refered.getDestTable()))); tableNode.setChild(0, newTableNode); - newTableNode.setParent(tableNode); ASTNode newColumnNode = new ASTNode(new CommonToken(HiveParser.Identifier, refered.getRefColumn())); node.setChild(1, newColumnNode); - newColumnNode.setParent(node); } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java index bcfc1f6..b253b94 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java @@ -44,7 +44,11 @@ abstract class DimHQLContext extends SimpleHQLContext { public CubeQueryContext getQuery() { return query; } - + DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery, + Set<Dimension> queriedDims, QueryAST ast) throws LensException { + this(query, dimsToQuery, queriedDims, ast.getSelectTree(), ast.getWhereTree(), ast.getGroupByTree(), + ast.getOrderByTree(), ast.getHavingTree(), ast.getLimitValue()); + } DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> queriedDims, String select, String where, String groupby, String orderby, String having, Integer limit) throws LensException { http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java index 0c43d98..d22287b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java @@ -32,16 +32,15 @@ import org.apache.lens.server.api.error.LensException; */ class DimOnlyHQLContext extends DimHQLContext { - DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query) throws LensException { - super(query, dimsToQuery, dimsToQuery.keySet(), query.getSelectTree(), - query.getWhereTree(), query.getGroupByTree(), query.getOrderByTree(), - query.getHavingTree(), query.getLimitValue()); + DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query, QueryAST ast) + throws LensException { + this(dimsToQuery, dimsToQuery.keySet(), query, ast); } - DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query, String whereClause) + DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> dimsQueried, + CubeQueryContext query, QueryAST ast) throws LensException { - super(query, dimsToQuery, dimsToQuery.keySet(), query.getSelectTree(), whereClause, query.getGroupByTree(), query - .getOrderByTree(), query.getHavingTree(), query.getLimitValue()); + super(query, dimsToQuery, dimsQueried, ast); } public String toHQL() throws LensException { @@ -49,7 +48,7 @@ class DimOnlyHQLContext extends DimHQLContext { } protected String getFromTable() throws LensException { - if (query.getAutoJoinCtx() != null && query.getAutoJoinCtx().isJoinsResolved()) { + if (query.isAutoJoinResolved()) { return getDimsToQuery().get(query.getAutoJoinCtx().getAutoJoinTarget()).getStorageString( query.getAliasForTableName(query.getAutoJoinCtx().getAutoJoinTarget().getName())); } else { http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index 776021d..26514d8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -432,7 +432,7 @@ class ExpressionResolver implements ContextRewriter { } public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, - boolean replaceFact) throws LensException { + QueryAST queryAST) throws LensException { Set<Dimension> exprDims = new HashSet<Dimension>(); if (!allExprsQueried.isEmpty()) { // pick expressions for fact @@ -446,7 +446,7 @@ class ExpressionResolver implements ContextRewriter { } } // Replace picked expressions in all the base trees - replacePickedExpressions(cfact, replaceFact); + replacePickedExpressions(queryAST); for (Set<PickedExpression> peSet : pickedExpressions.values()) { for (PickedExpression pe : peSet) { exprDims.addAll(pe.pickedCtx.exprDims); @@ -457,21 +457,13 @@ class ExpressionResolver implements ContextRewriter { return exprDims; } - private void replacePickedExpressions(CandidateFact cfact, boolean replaceFact) + private void replacePickedExpressions(QueryAST queryAST) throws LensException { - if (replaceFact) { - replaceAST(cubeql, cfact.getSelectAST()); - replaceAST(cubeql, cfact.getWhereAST()); - replaceAST(cubeql, cfact.getJoinTree()); - replaceAST(cubeql, cfact.getGroupByAST()); - replaceAST(cubeql, cfact.getHavingAST()); - } else { - replaceAST(cubeql, cubeql.getSelectAST()); - replaceAST(cubeql, cubeql.getWhereAST()); - replaceAST(cubeql, cubeql.getJoinTree()); - replaceAST(cubeql, cubeql.getGroupByAST()); - replaceAST(cubeql, cubeql.getHavingAST()); - } + replaceAST(cubeql, queryAST.getSelectAST()); + replaceAST(cubeql, queryAST.getWhereAST()); + replaceAST(cubeql, queryAST.getJoinAST()); + replaceAST(cubeql, queryAST.getGroupByAST()); + replaceAST(cubeql, queryAST.getHavingAST()); replaceAST(cubeql, cubeql.getOrderByAST()); } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java deleted file mode 100644 index 6c44233..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import java.util.Map; -import java.util.Set; - -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.server.api.error.LensException; - -import lombok.extern.slf4j.Slf4j; - -/** - * HQL context class which passes all query strings from the fact and works with required dimensions for the fact. - */ -@Slf4j -public class FactHQLContext extends DimHQLContext { - - private final CandidateFact fact; - private final Set<Dimension> factDims; - - FactHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> factDims, - CubeQueryContext query) throws LensException { - super(query, dimsToQuery, factDims, fact.getSelectTree(), fact.getWhereTree(), fact.getGroupByTree(), null, fact - .getHavingTree(), null); - this.fact = fact; - this.factDims = factDims; - log.info("factDims:{} for fact:{}", factDims, fact); - } - - @Override - protected Set<Dimension> getQueriedDimSet() { - return factDims; - } - - @Override - protected CandidateFact getQueriedFact() { - return fact; - } - - protected String getFromTable() throws LensException { - return query.getQBFromString(fact, getDimsToQuery()); - } - - public CandidateFact getFactToQuery() { - return fact; - } - -} http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java index da74713..9674f73 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java @@ -78,7 +78,6 @@ class GroupbyResolver implements ContextRewriter { if (groupbyAST != null) { // groupby ast exists, add the expression to AST groupbyAST.addChild(exprAST); - exprAST.setParent(groupbyAST); } else { // no group by ast exist, create one ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY)); @@ -153,7 +152,6 @@ class GroupbyResolver implements ContextRewriter { parent.setChild(i + 1, ch); } parent.setChild(index, child); - child.setParent(parent); } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 7cea7d5..6c3d4c3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -293,7 +293,6 @@ public final class HQLParser { if (original.getChildren() != null) { for (Object o : original.getChildren()) { ASTNode childCopy = copyAST((ASTNode) o); - childCopy.setParent(copy); copy.addChild(childCopy); } } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java index 1385584..de3a16e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java @@ -87,8 +87,8 @@ class JoinResolver implements ContextRewriter { boolean joinResolverDisabled = cubeql.getConf().getBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, CubeQueryConfUtil.DEFAULT_DISABLE_AUTO_JOINS); if (joinResolverDisabled) { - if (cubeql.getJoinTree() != null) { - cubeQB.setQbJoinTree(genJoinTree(cubeQB, cubeql.getJoinTree(), cubeql)); + if (cubeql.getJoinAST() != null) { + cubeQB.setQbJoinTree(genJoinTree(cubeql.getJoinAST(), cubeql)); } } else { autoResolveJoins(cubeql); @@ -336,7 +336,7 @@ class JoinResolver implements ContextRewriter { } // Recursively find out join conditions - private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, CubeQueryContext cubeql) throws LensException { + private QBJoinTree genJoinTree(ASTNode joinParseTree, CubeQueryContext cubeql) throws LensException { QBJoinTree joinTree = new QBJoinTree(); JoinCond[] condn = new JoinCond[1]; @@ -388,7 +388,7 @@ class JoinResolver implements ContextRewriter { } else if (isJoinToken(left)) { // Left subtree is join token itself, so recurse down - QBJoinTree leftTree = genJoinTree(qb, left, cubeql); + QBJoinTree leftTree = genJoinTree(left, cubeql); joinTree.setJoinSrc(leftTree); @@ -436,12 +436,9 @@ class JoinResolver implements ContextRewriter { return joinTree; } - private boolean isJoinToken(ASTNode node) { - if ((node.getToken().getType() == TOK_JOIN) || (node.getToken().getType() == TOK_LEFTOUTERJOIN) + private static boolean isJoinToken(ASTNode node) { + return (node.getToken().getType() == TOK_JOIN) || (node.getToken().getType() == TOK_LEFTOUTERJOIN) || (node.getToken().getType() == TOK_RIGHTOUTERJOIN) || (node.getToken().getType() == TOK_FULLOUTERJOIN) - || (node.getToken().getType() == TOK_LEFTSEMIJOIN) || (node.getToken().getType() == TOK_UNIQUEJOIN)) { - return true; - } - return false; + || (node.getToken().getType() == TOK_LEFTSEMIJOIN) || (node.getToken().getType() == TOK_UNIQUEJOIN); } } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java index 113d8de..1a729f8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java @@ -34,18 +34,24 @@ import com.google.common.collect.Lists; */ class MultiFactHQLContext extends SimpleHQLContext { - private Map<Dimension, CandidateDim> dimsToQuery; private Set<CandidateFact> facts; private CubeQueryContext query; - private Map<CandidateFact, Set<Dimension>> factDimMap; + private Map<CandidateFact, SimpleHQLContext> factHQLContextMap = new HashMap<>(); MultiFactHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery, Map<CandidateFact, Set<Dimension>> factDimMap, CubeQueryContext query) throws LensException { super(); this.query = query; this.facts = facts; - this.dimsToQuery = dimsToQuery; - this.factDimMap = factDimMap; + for (CandidateFact fact : facts) { + if (fact.getStorageTables().size() > 1) { + factHQLContextMap.put(fact, new SingleFactMultiStorageHQLContext(fact, dimsToQuery, query, fact)); + } else { + factHQLContextMap.put(fact, + new SingleFactSingleStorageHQLContext(fact, dimsToQuery, factDimMap.get(fact), query, + DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), fact))); + } + } } protected void setMissingExpressions() throws LensException { @@ -78,8 +84,7 @@ class MultiFactHQLContext extends SimpleHQLContext { } private String getSelectString() throws LensException { - Map<Integer, List<Integer>> selectToFactIndex = - new HashMap<Integer, List<Integer>>(query.getSelectAST().getChildCount()); + Map<Integer, List<Integer>> selectToFactIndex = new HashMap<>(query.getSelectAST().getChildCount()); int fi = 1; for (CandidateFact fact : facts) { for (int ind : fact.getSelectIndices()) { @@ -116,33 +121,14 @@ class MultiFactHQLContext extends SimpleHQLContext { return select.toString(); } - public Map<Dimension, CandidateDim> getDimsToQuery() { - return dimsToQuery; - } - - public Set<CandidateFact> getFactsToQuery() { - return facts; - } - private String getFromString() throws LensException { StringBuilder fromBuilder = new StringBuilder(); int aliasCount = 1; - Iterator<CandidateFact> iter = facts.iterator(); - while (iter.hasNext()) { - CandidateFact fact = iter.next(); - if (fact.getStorageTables().size() > 1) { - // Not supported right now. - throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo()); - } - FactHQLContext facthql = new FactHQLContext(fact, dimsToQuery, factDimMap.get(fact), query); - fromBuilder.append("("); - fromBuilder.append(facthql.toHQL()); - fromBuilder.append(")"); - fromBuilder.append(" mq" + aliasCount); - aliasCount++; - if (iter.hasNext()) { - fromBuilder.append(" full outer join "); - } + String sep = ""; + for (CandidateFact fact : facts) { + SimpleHQLContext facthql = factHQLContextMap.get(fact); + fromBuilder.append(sep).append("(").append(facthql.toHQL()).append(")").append(" mq").append(aliasCount++); + sep = " full outer join "; } CandidateFact firstFact = facts.iterator().next(); if (!firstFact.getDimFieldIndices().isEmpty()) { http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java new file mode 100644 index 0000000..31680ca --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + + +import org.apache.hadoop.hive.ql.parse.ASTNode; + + +interface QueryAST { + + String getSelectTree(); + + String getWhereTree(); + + String getHavingTree(); + + String getOrderByTree(); + + String getGroupByTree(); + + Integer getLimitValue(); + + void setLimitValue(Integer integer); + + /** + * @return the selectAST + */ + + ASTNode getSelectAST(); + + /** + * @param selectAST the selectAST to set + */ + + void setSelectAST(ASTNode selectAST); + + /** + * @return the whereAST + */ + + ASTNode getWhereAST(); + + /** + * @param whereAST the whereAST to set + */ + + void setWhereAST(ASTNode whereAST); + + /** + * @return the havingAST + */ + + ASTNode getHavingAST(); + + /** + * @param havingAST the havingAST to set + */ + + void setHavingAST(ASTNode havingAST); + + ASTNode getGroupByAST(); + + void setGroupByAST(ASTNode havingAST); + + ASTNode getJoinAST(); + + ASTNode getOrderByAST(); + + void setOrderByAST(ASTNode node); +} http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java deleted file mode 100644 index de52b0a..0000000 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.cube.parse; - -import java.util.Map; - -import org.apache.lens.cube.metadata.Dimension; -import org.apache.lens.cube.metadata.TimeRange; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.ql.parse.ASTNode; - -/** - * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being - * queried. - * <p/> - * Updates from string with join clause expanded - */ -class SingleFactHQLContext extends DimOnlyHQLContext { - - private final CandidateFact fact; - private String storageAlias; - - SingleFactHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query) - throws LensException { - super(dimsToQuery, query); - this.fact = fact; - } - - SingleFactHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery, - CubeQueryContext query, String whereClause) throws LensException { - super(dimsToQuery, query, whereClause); - this.fact = fact; - this.storageAlias = storageAlias; - } - - - public CandidateFact getFactToQuery() { - return fact; - } - - static void addRangeClauses(CubeQueryContext query, CandidateFact fact) throws LensException { - if (fact != null) { - // resolve timerange positions and replace it by corresponding where - // clause - for (TimeRange range : query.getTimeRanges()) { - for (Map.Entry<String, String> entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) { - String table = entry.getValue(); - String rangeWhere = entry.getKey(); - - if (!StringUtils.isBlank(rangeWhere)) { - ASTNode rangeAST = HQLParser.parseExpr(rangeWhere); - rangeAST.setParent(range.getParent()); - range.getParent().setChild(range.getChildIndex(), rangeAST); - } - fact.getStorgeWhereClauseMap().put(table, query.getWhereTree()); - } - } - } - } - - - @Override - protected String getFromTable() throws LensException { - if (getQuery().getAutoJoinCtx() != null && getQuery().getAutoJoinCtx().isJoinsResolved()) { - if (storageAlias != null) { - return storageAlias; - } else { - return fact.getStorageString(getQuery().getAliasForTableName(getQuery().getCube().getName())); - } - } else { - if (fact.getStorageTables().size() == 1) { - return getQuery().getQBFromString(fact, getDimsToQuery()); - } else { - return storageAlias; - } - } - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 96b1d05..e531e6b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_ENABLE_STORAGES_UNION; +import static org.apache.lens.cube.parse.CubeQueryConfUtil.ENABLE_STORAGES_UNION; import static org.apache.lens.cube.parse.HQLParser.*; import static org.apache.hadoop.hive.ql.parse.HiveParser.*; @@ -27,6 +29,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Dimension; import org.apache.lens.server.api.error.LensException; @@ -42,6 +45,7 @@ import lombok.Data; public class SingleFactMultiStorageHQLContext extends UnionHQLContext { + private final QueryAST ast; int aliasCounter = 0; @Data @@ -81,29 +85,33 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { private Map<HashableASTNode, ASTNode> innerToOuterASTs = new HashMap<>(); - SingleFactMultiStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query) + SingleFactMultiStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, + CubeQueryContext query, QueryAST ast) throws LensException { super(query, fact); + if (!query.getConf().getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) { + throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo()); + } + this.ast = ast; processSelectAST(); processGroupByAST(); processWhereAST(); processHavingAST(); processOrderByAST(); processLimit(); - setHqlContexts(getUnionContexts(fact, dimsToQuery, query)); + setHqlContexts(getUnionContexts(fact, dimsToQuery, query, ast)); } private void processSelectAST() { - query.getSelectFinalAliases().clear(); - ASTNode originalSelectAST = copyAST(query.getSelectAST()); - query.setSelectAST(new ASTNode(originalSelectAST.getToken())); + ASTNode originalSelectAST = copyAST(ast.getSelectAST()); + ast.setSelectAST(new ASTNode(originalSelectAST.getToken())); ASTNode outerSelectAST = processExpression(originalSelectAST); setSelect(getString(outerSelectAST)); } private void processGroupByAST() { - if (query.getGroupByAST() != null) { - setGroupby(getString(processExpression(query.getGroupByAST()))); + if (ast.getGroupByAST() != null) { + setGroupby(getString(processExpression(ast.getGroupByAST()))); } } @@ -111,28 +119,29 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { for (String storageTable : fact.getStorgeWhereClauseMap().keySet()) { ASTNode tree = parseExpr(fact.getStorgeWhereClauseMap().get(storageTable)); ASTNode replaced = replaceAST(tree); + //TODO: optimize parse/unparse cycle fact.getStorgeWhereClauseMap().put(storageTable, getString(replaced)); } } private void processHavingAST() throws LensException { - if (query.getHavingAST() != null) { - setHaving(getString(processExpression(query.getHavingAST()))); - query.setHavingAST(null); + if (ast.getHavingAST() != null) { + setHaving(getString(processExpression(ast.getHavingAST()))); + ast.setHavingAST(null); } } private void processOrderByAST() { - if (query.getOrderByAST() != null) { - setOrderby(getString(processExpression(query.getOrderByAST()))); - query.setOrderByAST(null); + if (ast.getOrderByAST() != null) { + setOrderby(getString(processExpression(ast.getOrderByAST()))); + ast.setOrderByAST(null); } } private void processLimit() { - setLimit(query.getLimitValue()); - query.setLimitValue(null); + setLimit(ast.getLimitValue()); + ast.setLimitValue(null); } /* @@ -171,6 +180,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { addToInnerSelectAST(innerSelectExprAST); ASTNode dotAST = getDotAST(query.getCube().getName(), alias); ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION)); + //TODO: take care or non-transitive aggregate functions outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText()))); outerAST.addChild(dotAST); innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); @@ -225,10 +235,10 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } private void addToInnerSelectAST(ASTNode selectExprAST) { - if (query.getSelectAST() == null) { - query.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT))); + if (ast.getSelectAST() == null) { + ast.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT))); } - query.getSelectAST().addChild(selectExprAST); + ast.getSelectAST().addChild(selectExprAST); } private ASTNode getDotAST(String tableAlias, String fieldAlias) { @@ -245,13 +255,13 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { } private static ArrayList<HQLContextInterface> getUnionContexts(CandidateFact fact, Map<Dimension, CandidateDim> - dimsToQuery, CubeQueryContext query) + dimsToQuery, CubeQueryContext query, QueryAST ast) throws LensException { ArrayList<HQLContextInterface> contexts = new ArrayList<>(); String alias = query.getAliasForTableName(query.getCube().getName()); for (String storageTable : fact.getStorageTables()) { - SingleFactHQLContext ctx = new SingleFactHQLContext(fact, storageTable + " " + alias, dimsToQuery, query, - fact.getWhereClause(storageTable.substring(storageTable.indexOf(".") + 1))); + SingleFactSingleStorageHQLContext ctx = new SingleFactSingleStorageHQLContext(fact, storageTable + " " + alias, + dimsToQuery, query, DefaultQueryAST.fromCandidateFact(fact, storageTable, ast)); contexts.add(ctx); } return contexts; http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java new file mode 100644 index 0000000..b1a3b3f --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.Map; +import java.util.Set; + +import org.apache.lens.cube.metadata.Dimension; +import org.apache.lens.server.api.error.LensException; + +/** + * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being + * queried. + * <p/> + * Updates from string with join clause expanded + */ +class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { + + private final CandidateFact fact; + private final Set<Dimension> queriedDimSet; + private String storageAlias; + + SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, + CubeQueryContext query, QueryAST ast) + throws LensException { + this(fact, dimsToQuery, dimsToQuery.keySet(), query, ast); + } + + SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, + Set<Dimension> dimsQueried, CubeQueryContext query, QueryAST ast) + throws LensException { + super(dimsToQuery, dimsQueried, query, ast); + this.fact = fact; + this.queriedDimSet = dimsQueried; + } + + SingleFactSingleStorageHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery, + CubeQueryContext query, QueryAST ast) throws LensException { + this(fact, dimsToQuery, query, ast); + this.storageAlias = storageAlias; + } + + @Override + protected String getFromTable() throws LensException { + if (getQuery().isAutoJoinResolved()) { + if (storageAlias != null) { + return storageAlias; + } else { + return fact.getStorageString(query.getAliasForTableName(query.getCube().getName())); + } + } else { + if (fact.getStorageTables().size() == 1) { + return getQuery().getQBFromString(fact, getDimsToQuery()); + } else { + return storageAlias; + } + } + } + + @Override + protected CandidateFact getQueriedFact() { + return fact; + } + + @Override + public Set<Dimension> getQueriedDimSet() { + return queriedDimSet; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 62cc071..14def15 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -19,14 +19,10 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateUtil.WSPACE; -import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName; -import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableEndTimesKey; -import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableStartTimesKey; +import static org.apache.lens.cube.metadata.MetastoreUtil.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST; -import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.RANGE_NOT_ANSWERABLE; -import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.*; import java.text.DateFormat; import java.text.ParseException; @@ -36,10 +32,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; -import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause; -import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode; -import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode; +import org.apache.lens.cube.parse.CandidateTablePruneCause.*; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -49,7 +42,6 @@ import org.apache.hadoop.util.ReflectionUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; /** @@ -361,9 +353,9 @@ class StorageTableResolver implements ContextRewriter { private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws LensException { // Find candidate tables wrt supported storages Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); - Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>(); while (i.hasNext()) { CandidateFact cfact = i.next(); + Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>(); List<FactPartition> answeringParts = new ArrayList<>(); Map<String, SkipStorageCause> skipStorageCauses = skipStorageCausesPerFact.get(cfact.fact); if (skipStorageCauses == null) { @@ -434,9 +426,6 @@ class StorageTableResolver implements ContextRewriter { cfact.incrementPartsQueried(rangeParts.size()); answeringParts.addAll(rangeParts); cfact.getPartsQueried().addAll(rangeParts); - String rangeWhereClause = rangeWriter.getTimeRangeWhereClause(cubeql, - cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts); - cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString())); } if (!unsupportedTimeDims.isEmpty()) { log.info("Not considering fact table:{} as it doesn't support time dimensions: {}", cfact.fact, @@ -483,24 +472,27 @@ class StorageTableResolver implements ContextRewriter { Set<String> storageTables = new LinkedHashSet<>(); storageTables.addAll(minimalStorageTables.keySet()); cfact.setStorageTables(storageTables); - // Update range->storage->partitions with time range where clause for (TimeRange trange : cfact.getRangeToStoragePartMap().keySet()) { - Map<String, String> rangeToWhere = new HashMap<String, String>(); + Map<String, String> rangeToWhere = new HashMap<>(); for (Map.Entry<String, Set<FactPartition>> entry : minimalStorageTables.entrySet()) { String table = entry.getKey(); Set<FactPartition> minimalParts = entry.getValue(); LinkedHashSet<FactPartition> rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table); - LinkedHashSet<FactPartition> minimalPartsCopy = new LinkedHashSet<FactPartition>(minimalParts); - minimalPartsCopy.retainAll(rangeParts); + LinkedHashSet<FactPartition> minimalPartsCopy = Sets.newLinkedHashSet(); + + if (rangeParts != null) { + minimalPartsCopy.addAll(minimalParts); + minimalPartsCopy.retainAll(rangeParts); + } if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) { - rangeToWhere.put( - rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), - minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table); + rangeToWhere.put(table, "((" + + rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), + minimalPartsCopy) + ") and (" + whereClauseForFallback.get(trange) + "))"); } else { - rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql, - cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table); + rangeToWhere.put(table, rangeWriter.getTimeRangeWhereClause(cubeql, + cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy)); } } cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere); @@ -592,8 +584,7 @@ class StorageTableResolver implements ContextRewriter { int lookAheadNumParts = conf.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(interval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS); - TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1) - .iterator(); + TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1).iterator(); // add partitions from ceilFrom to floorTo while (iter.hasNext()) { Date dt = iter.next(); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 87e4ce3..e7b9403 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -26,6 +26,10 @@ import java.util.Calendar; import java.util.Date; import java.util.HashMap; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.Lists; + public class DateFactory { private DateFactory() { @@ -143,6 +147,7 @@ public class DateFactory { // Time Ranges public static final String LAST_HOUR_TIME_RANGE; public static final String TWO_DAYS_RANGE; + public static final String TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS; public static final String TWO_DAYS_RANGE_TTD; public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS; public static final String TWO_DAYS_RANGE_TTD2; @@ -192,5 +197,11 @@ public class DateFactory { // calculate LAST_HOUR_TIME_RANGE LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0); + + TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS = StringUtils.join(" OR ", Lists.newArrayList( + getTimeRangeString(getDateStringWithOffset(HOURLY, -48), getDateStringWithOffset(DAILY, -1)), + getTimeRangeString(getDateStringWithOffset(DAILY, 0), getDateStringWithOffset(HOURLY, 0)), + getTimeRangeString(getDateStringWithOffset(DAILY, -1), getDateStringWithOffset(DAILY, 0)) + )); } } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 3f01dbe..ad20ae1 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -512,7 +512,7 @@ public class CubeTestSetup { "No aggregateMsr", null, null, null)); cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"), "New measure", null, null, null, NOW, null, 100.0)); - cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "first measure"), "Measure15", null, "SUM", + cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "fifteenth measure"), "Measure15", null, "SUM", "RS")); cubeDimensions = new HashSet<CubeDimAttribute>(); http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index a5886dc..5b44f95 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -50,6 +50,7 @@ import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import com.google.common.base.Splitter; +import com.google.common.collect.Sets; import lombok.Getter; public class TestBaseCubeQueries extends TestQueryRewrite { @@ -546,17 +547,24 @@ public class TestBaseCubeQueries extends TestQueryRewrite { assertEquals(ctx.getCandidateFactSets().size(), 1); assertEquals(ctx.getCandidateFactSets().iterator().next().size(), 1); CandidateFact cfact = ctx.getCandidateFactSets().iterator().next().iterator().next(); - assertEquals(cfact.getRangeToWhereClause().size(), 2); - for(Map.Entry<TimeRange, String> entry: cfact.getRangeToWhereClause().entrySet()) { + + assertEquals(cfact.getRangeToStoragePartMap().size(), 2); + Set<String> storages = Sets.newHashSet(); + for(Map<String, String> entry: cfact.getRangeToStorageWhereMap().values()) { + storages.addAll(entry.keySet()); + } + assertEquals(storages.size(), 1); + String storage = storages.iterator().next(); + for(Map.Entry<TimeRange, Map<String, String>> entry: cfact.getRangeToStorageWhereMap().entrySet()) { if (entry.getKey().getPartitionColumn().equals("dt")) { - ASTNode parsed = HQLParser.parseExpr(entry.getValue()); + ASTNode parsed = HQLParser.parseExpr(entry.getValue().get(storage)); assertEquals(parsed.getToken().getType(), KW_AND); - assertTrue(entry.getValue().substring(((CommonToken) parsed.getToken()).getStopIndex() + 1).toLowerCase() - .contains(dTimeWhereClause)); - assertFalse(entry.getValue().substring(0, ((CommonToken) parsed.getToken()).getStartIndex()).toLowerCase() - .contains("and")); + assertTrue(entry.getValue().get(storage).substring(((CommonToken) parsed.getToken()).getStopIndex() + 1) + .toLowerCase().contains(dTimeWhereClause)); + assertFalse(entry.getValue().get(storage).substring(0, ((CommonToken) parsed.getToken()).getStartIndex()) + .toLowerCase().contains("and")); } else if (entry.getKey().getPartitionColumn().equals("ttd")) { - assertFalse(entry.getValue().toLowerCase().contains("and")); + assertFalse(entry.getValue().get(storage).toLowerCase().contains("and")); } else { throw new LensException("Unexpected"); } http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 9a08735..f02cdb0 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -41,7 +41,6 @@ import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.time.DateUtils; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -58,7 +57,6 @@ import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; @Slf4j @@ -410,7 +408,7 @@ public class TestCubeRewriter extends TestQueryRewrite { return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage); } }; - try{ + try { rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` " + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf); fail("Union feature is disabled, should have failed"); @@ -501,6 +499,35 @@ public class TestCubeRewriter extends TestQueryRewrite { } @Test + public void testMultiFactMultiStorage() throws ParseException, LensException { + Configuration conf = LensServerAPITestUtil.getConfiguration( + CubeQueryConfUtil.ENABLE_STORAGES_UNION, true, + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C2", + getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact", + getValidUpdatePeriodsKey("testfact", "C1"), "HOURLY", + getValidUpdatePeriodsKey("testfact", "C2"), "DAILY", + getValidUpdatePeriodsKey("testfact2_raw", "C1"), "YEARLY", + getValidUpdatePeriodsKey("testfact2_raw", "C2"), "YEARLY"); + CubeTestSetup.getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY)); + CubeTestSetup.getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(DAILY)); + String whereCond = "zipcode = 'a' and cityid = 'b' and (" + TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS + ")"; + String hqlQuery = rewrite("cube select zipcode, count(msr4), sum(msr15) from testCube where " + whereCond, conf); + System.out.println(hqlQuery); + String possibleStart1 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq1.msr4 msr4, mq2.msr15 msr15 FROM "; + String possibleStart2 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq2.msr4 msr4, mq1.msr15 msr15 FROM "; + + assertTrue(hqlQuery.startsWith(possibleStart1) || hqlQuery.startsWith(possibleStart2)); + compareContains(rewrite("cube select zipcode as `zipcode`, sum(msr15) as `msr15` from testcube where " + whereCond, + conf), hqlQuery); + compareContains(rewrite("cube select zipcode as `zipcode`, count(msr4) as `msr4` from testcube where " + whereCond, + conf), hqlQuery); + assertTrue(hqlQuery.endsWith("on mq1.zipcode <=> mq2.zipcode")); + // No time_range_in should be remaining + assertFalse(hqlQuery.contains("time_range_in")); + //TODO: handle having after LENS-813, also handle for order by and limit + } + + @Test public void testCubeWhereQueryWithMultipleTables() throws Exception { Configuration conf = getConf(); conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true); @@ -1120,18 +1147,18 @@ public class TestCubeRewriter extends TestQueryRewrite { new HashMap<String, List<CandidateTablePruneCause>>() { { put("statetable", Arrays.asList(CandidateTablePruneCause.noCandidateStorages( - new HashMap<String, SkipStorageCause>() { - { - put("c1_statetable", new SkipStorageCause(SkipStorageCode.NO_PARTITIONS)); - } - })) + new HashMap<String, SkipStorageCause>() { + { + put("c1_statetable", new SkipStorageCause(SkipStorageCode.NO_PARTITIONS)); + } + })) ); put("statetable_partitioned", Arrays.asList(CandidateTablePruneCause.noCandidateStorages( - new HashMap<String, SkipStorageCause>() { - { - put("C3_statetable_partitioned", new SkipStorageCause(SkipStorageCode.UNSUPPORTED)); - } - })) + new HashMap<String, SkipStorageCause>() { + { + put("C3_statetable_partitioned", new SkipStorageCause(SkipStorageCode.UNSUPPORTED)); + } + })) ); } }
