[04/12] lens git commit: LENS-1444: Optimize the algorithm of finding all eligible union candidates

2017-07-13 Thread prongs 
LENS-1444: Optimize the algorithm of finding all eligible union candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7f407bc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7f407bc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7f407bc

Branch: refs/heads/current-release-line
Commit: a7f407bcb59ffa84c8ab6e830ba98aee81516085
Parents: c174583
Author: Rajat Khandelwal 
Authored: Fri Jun 23 16:39:37 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:52 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java  | 17 -
 1 file changed, 4 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a7f407bc/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 61c28c6..8e07162 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -106,16 +106,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return true;
   }
 
-  private void pruneUnionCandidatesNotCoveringAllRanges(List 
ucs, CubeQueryContext cubeql) {
-for (Iterator itr = ucs.iterator(); itr.hasNext();) {
-  UnionCandidate uc = itr.next();
-  if (!isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
-itr.remove();
-cubeql.addCandidatePruningMsg(uc, 
CandidateTablePruneCause.storageNotAvailableInRange(cubeql.getTimeRanges()));
-  }
-}
-  }
-
   private List resolveTimeRangeCoveringFactSet(CubeQueryContext 
cubeql,
   Set queriedMsrs, List 
qpcList) throws LensException {
 List candidateSet = new ArrayList<>();
@@ -138,8 +128,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
-// prune non covering sets
-pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql);
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
@@ -197,7 +185,10 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 clonedI = clonedI >>> 1;
 --count;
   }
-  combinations.add(new UnionCandidate(individualCombinationList, cubeql));
+  UnionCandidate uc = new UnionCandidate(individualCombinationList, 
cubeql);
+  if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+combinations.add(uc);
+  }
 }
 return combinations;
   }

[02/12] lens git commit: LENS-1442: Optimize algorithm of CandidateCoveringSetResolver

2017-07-13 Thread prongs 
LENS-1442: Optimize algorithm of CandidateCoveringSetResolver


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9da5b40e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9da5b40e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9da5b40e

Branch: refs/heads/current-release-line
Commit: 9da5b40e3afbe966256a601bae6204afd5d3e992
Parents: c2a9c93
Author: Rajat Khandelwal 
Authored: Tue Jun 20 16:09:32 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:49 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java| 15 +--
 1 file changed, 9 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/9da5b40e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1e9873f..61c28c6 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -208,21 +208,24 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 List ucSet = new ArrayList<>(candidates);
 // Check if a single set can answer all the measures and exprsWithMeasures
 for (Iterator i = ucSet.iterator(); i.hasNext();) {
-  boolean evaluable = false;
+  boolean allEvaluable = true;
+  boolean anyEvaluable = false;
   Candidate uc = i.next();
   for (QueriedPhraseContext msr : msrs) {
-evaluable = uc.isPhraseAnswerable(msr);
-if (!evaluable) {
-  break;
-}
+boolean evaluable = uc.isPhraseAnswerable(msr);
+allEvaluable &= evaluable;
+anyEvaluable |= evaluable;
   }
-  if (evaluable) {
+  if (allEvaluable) {
 // single set can answer all the measures as an UnionCandidate
 List one = new ArrayList<>();
 one.add(uc);
 msrCoveringSets.add(one);
 i.remove();
   }
+  if (!anyEvaluable) { // none evaluable
+i.remove();
+  }
 }
 // Sets that contain all measures or no measures are removed from 
iteration.
 // find other facts

[09/12] lens git commit: LENS-1450: filters are getting replicated in jdbc driver query

2017-07-13 Thread prongs 
LENS-1450: filters are getting replicated in jdbc driver query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/503a46dc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/503a46dc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/503a46dc

Branch: refs/heads/current-release-line
Commit: 503a46dc6c89598cc2ea557f2311e4bd26c3d9a1
Parents: dd3b1bd
Author: Rajitha R 
Authored: Wed Jul 5 16:49:07 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:58 2017 +0530

--
 .../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/503a46dc/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b5330a7..fb49b6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,7 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 }
   }
 } else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND) {
+&& node.getToken().getType() != HiveParser.KW_AND && 
node.getChildCount() == 0) {
   // if node is the only child
   allFilters.add(HQLParser.getString((ASTNode) node));
 }

[10/12] lens git commit: LENS-1449 : lens.query.result.parent.dir to be set at driver level.

2017-07-13 Thread prongs 
LENS-1449 : lens.query.result.parent.dir to be set at driver level.


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/41051ea4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/41051ea4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/41051ea4

Branch: refs/heads/current-release-line
Commit: 41051ea47f164f004f5fccaa7ea91765007811fc
Parents: 503a46d
Author: Raghavendra Singh 
Authored: Thu Jul 6 14:03:59 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:59 2017 +0530

--
 .../api/query/DriverSelectorQueryContext.java   |  4 +++-
 .../lens/server/api/query/QueryContext.java |  5 +
 .../lens/server/query/TestQueryService.java |  6 --
 .../lens/server/query/TestResultFormatting.java | 20 +++-
 .../drivers/hive/hive1/hivedriver-site.xml  |  5 +
 .../drivers/hive/hive2/hivedriver-site.xml  |  6 ++
 .../drivers/jdbc/jdbc1/jdbcdriver-site.xml  |  5 +
 lens-server/src/test/resources/lens-site.xml|  5 -
 8 files changed, 47 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 8e431d1..41e533e 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -155,7 +155,9 @@ public class DriverSelectorQueryContext {
   private Configuration mergeConf(LensDriver driver, Configuration queryConf) {
 Configuration conf = new Configuration(driver.getConf());
 for (Map.Entry entry : queryConf) {
-  conf.set(entry.getKey(), entry.getValue());
+  if (!conf.getFinalParameters().contains(entry.getKey())) {
+conf.set(entry.getKey(), entry.getValue());
+  }
 }
 conf.setClassLoader(queryConf.getClassLoader());
 return conf;

http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 9923589..ccdef87 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -328,6 +328,11 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   }
 
   public String getResultSetParentDir() {
+if (getSelectedDriver() != null && 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) != null) {
+  log.info("Fetching Parent Dir from driver conf:- "
+  + 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR));
+  return 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR);
+}
 return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, 
LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT);
   }
 

http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java 
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 1149696..b7cdb88 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -848,9 +848,11 @@ public class TestQueryService extends LensJerseyTest {
 LensResultSet rs = queryService.getResultset(handle3);
 //check persisted result path
 String expectedPath =
-ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + 
handle3.getHandleIdString()
+
ctx3.getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" 
+ handle3.getHandleIdString()
 + ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN);
-assertTrue(((PersistentResultSet) 
rs).getOutputPath().endsWith(expectedPath));
+assertTrue(((PersistentResultSet) 
rs).getOutputPath().endsWith(expectedPath)
+, "Result

[05/12] lens git commit: LENS-1445: Expression having reference column ends up rewriting wrong query

2017-07-13 Thread prongs 
LENS-1445: Expression having reference column ends up rewriting wrong query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d49f45a0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d49f45a0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d49f45a0

Branch: refs/heads/current-release-line
Commit: d49f45a0f8c6665784a3770a534d6495c21fd1bc
Parents: a7f407b
Author: Sushil Mohanty 
Authored: Fri Jun 23 16:40:25 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:53 2017 +0530

--
 .../lens/cube/parse/UnionQueryWriter.java   | 79 +---
 .../parse/TestCubeSegmentationRewriter.java |  3 +-
 .../cube/parse/TestUnionAndJoinCandidates.java  | 26 +++
 .../resources/schema/cubes/base/basecube.xml| 11 +++
 .../resources/schema/cubes/base/testcube.xml|  1 +
 5 files changed, 92 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index cc0a2e5..f6c9ce1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -28,7 +28,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
 import java.util.*;
 
-import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -117,7 +118,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
* @return ASTNode
* @throws LensException
*/
-  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidate sc)
+  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
 if (cubeql.getHavingAST() != null) {
   ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
@@ -251,25 +252,15 @@ public class UnionQueryWriter extends SimpleHQLContext {
 for (StorageCandidateHQLContext sc : storageCandidates) {
   node = (ASTNode) 
sc.getQueryAst().getSelectAST().getChild(position).getChild(0);
   if (HQLParser.isAggregateAST(node) || HQLParser.hasAggregate(node)) {
-return MetastoreUtil.copyAST(node);
+if (!node.getChild(1).toString().equals(DEFAULT_MEASURE))  {
+  return MetastoreUtil.copyAST(node);
+}
   }
 }
 return MetastoreUtil.copyAST(node);
   }
 
   /**
-   *  Check if ASTNode is answerable by StorageCandidate
-   * @param sc
-   * @param node
-   * @return
-   */
-  private boolean isNodeNotAnswerableForStorageCandidate(StorageCandidate sc, 
ASTNode node) {
-Set cols = new LinkedHashSet<>();
-getAllColumnsOfNode(node, cols);
-return !sc.getColumns().containsAll(cols);
-  }
-
-  /**
* Set the default value "0.0" in the non answerable aggreagte expressions.
* @param node
* @param sc
@@ -467,7 +458,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
 // Iterate over the StorageCandidates and add non projected having columns 
in inner select ASTs
 for (StorageCandidateHQLContext sc : storageCandidates) {
   aliasDecider.setCounter(selectAliasCounter);
-  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, 
sc.getStorageCandidate());
+  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc);
 }
 removeRedundantProjectedPhrases();
   }
@@ -493,7 +484,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
   ASTNode child = (ASTNode) selectAST.getChild(i);
   ASTNode outerSelect = new ASTNode(child);
   ASTNode selectExprAST = (ASTNode) child.getChild(0);
-  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc.getStorageCandidate(), true,
+  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc, true,
   cubeql.getBaseCube().getDimAttributeNames());
   outerSelect.addChild(outerAST);
   // has an alias? add it
@@ -529,13 +520,14 @@ public class UnionQueryWriter extends SimpleHQLContext {
5. If given ast is memorized as mentioned in the above cases, return the 
mapping.
  */
   private ASTNode getOuterAST(ASTNode astNode, ASTNode innerSelectAST,
-  AliasDecider aliasDecider, StorageCandidate sc, boolean isSelectAst, 
Set dimensionSet)
+

[12/12] lens git commit: LENS-1452: Optimize Time Union candidate Algorithm

2017-07-13 Thread prongs 
LENS-1452: Optimize Time Union candidate Algorithm


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6dca4466
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6dca4466
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6dca4466

Branch: refs/heads/current-release-line
Commit: 6dca44661bf604ca1436c6cd1d3998405d0333a4
Parents: 3769ef0
Author: Rajat Khandelwal 
Authored: Mon Jul 10 16:52:54 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:43:03 2017 +0530

--
 .../parse/CandidateCoveringSetsResolver.java| 47 +---
 1 file changed, 41 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/6dca4466/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 8e07162..69d9562 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -26,6 +26,7 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.TimeRange;
 import org.apache.lens.server.api.error.LensException;
 
+import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -124,16 +125,14 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
 // Get all covering fact sets
-List unionCoveringSet =
-getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
+//List unionCoveringSet = getCombinations(new 
ArrayList<>(allCandidatesPartiallyValid), cubeql);
+List unionCoveringSet = 
getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
 }
-// prune redundant covering sets
-pruneRedundantUnionCoveringSets(unionCoveringSet);
 // pruing done in the previous steps, now create union candidates
 candidateSet.addAll(unionCoveringSet);
 updateQueriableMeasures(candidateSet, qpcList);
@@ -155,7 +154,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private void pruneRedundantUnionCoveringSets(List 
candidates) {
 for (int i = 0; i < candidates.size(); i++) {
   UnionCandidate current = candidates.get(i);
@@ -168,7 +167,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private List getCombinations(final List 
candidates, CubeQueryContext cubeql) {
 List combinations = new LinkedList<>();
 int size = candidates.size();
@@ -193,6 +192,42 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return combinations;
   }
 
+  /**
+   * The following function is iterative rewrite of the following 
tail-recursive implementation:
+   * (ignoring cubeql for clarity)
+   * getCombinations(candidates) = getCombinationsTailRecursive(emptyList(), 
candidates)
+   *
+   * getCombinationsTailRecursive(incompleteCombinations: 
List[List[Candidate]], candidates: List[Candidate]) =
+   *   head, tail = head and tail of linked List candidates
+   *   add head to all elements of incompleteCombinations.
+   *   complete = remove now complete combinations from incompleteCombinations
+   *   return complete ++ getCombinationsTailRecursive(incompleteCombinations, 
tail)
+   * @param candidates
+   * @param cubeql
+   * @return
+   */
+  private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
+LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
+List incompleteCombinations = Lists.newArrayList();
+List unionCandidates = Lists.newArrayList();
+
+while(!candidateLinkedList.isEmpty()) {
+  Candidate candidate = candidateLinkedList.remove();
+  incompleteCombinations.add(Lists.newArrayList());
+  Iterator iter = incompleteCombinations.iterator();
+  while(iter.hasNext()) {
+List incompleteCombination = iter.next();
+incompleteCombination.add(candidate);
+UnionCandidate unionCandidate = new 
UnionCandidate(incompleteCombination, cubeql);
+if

[03/12] lens git commit: LENS-1443: Fallback ranges not working for virtual facts

2017-07-13 Thread prongs 
LENS-1443: Fallback ranges not working for virtual facts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c174583f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c174583f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c174583f

Branch: refs/heads/current-release-line
Commit: c174583ff7946d0347e0a0a87272f42c0023aecf
Parents: 9da5b40
Author: Rajat Khandelwal 
Authored: Wed Jun 21 13:20:09 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:50 2017 +0530

--
 .../apache/lens/cube/metadata/FactTable.java| 38 ++--
 .../apache/lens/cube/parse/JoinCandidate.java   |  7 ++--
 .../lens/cube/parse/LeastPartitionResolver.java | 21 +--
 .../lens/cube/parse/StorageCandidate.java   |  2 +-
 .../lens/cube/parse/StorageTableResolver.java   |  5 +--
 5 files changed, 37 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index f87cf44..a463c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -35,132 +35,132 @@ public interface FactTable extends Named {
*
* @return Map of storage to set of update periods
*/
-  public Map getUpdatePeriods();
+  Map getUpdatePeriods();
 
   /**
* Cube to which this fact belongs to
*
* @return the cube string
*/
-  public String getCubeName();
+  String getCubeName();
 
   /**
* The set of Storage names
*
* @return set of strings
*/
-  public Set getStorages();
+  Set getStorages();
 
   /**
*The type of the fact
*
* @return table type {@link CubeTableType}
*/
-  public CubeTableType getTableType();
+  CubeTableType getTableType();
 
   /**
* Config properties
*
* @return map of string, string
*/
-  public Map getProperties();
+  Map getProperties();
 
   /**
* Valid columns of the fact
*
* @return list of column names
*/
-  public Set getValidColumns();
+  Set getValidColumns();
 
   /**
* Weight of the fact
*
* @return weight of the fact in double
*/
-  public double weight();
+  double weight();
 
   /**
* Set of all the columns names of the fact
*
* @return set of column names
*/
-  public Set getAllFieldNames();
+  Set getAllFieldNames();
 
   /**
*tag for checking data completeness
*
* @return Tag String
*/
-  public String getDataCompletenessTag();
+  String getDataCompletenessTag();
 
   /**
* List of columns of the fact
*
* @return set of {@link FieldSchema}
*/
-  public List getColumns();
+  List getColumns();
 
   /**
* Is Aggregated Fact
*
* @return true if fact is Aggregated , false otherwise
*/
-  public boolean isAggregated();
+  boolean isAggregated();
 
   /**
* Absolute start time of the fact
*
* @return Absolute Start time of the fact {@link Date}
*/
-  public Date getAbsoluteStartTime();
+  Date getAbsoluteStartTime();
 
   /**
* Relative start time of the fact
*
* @return Relative Start time of the fact {@link Date}
*/
-  public Date getRelativeStartTime();
+  Date getRelativeStartTime();
 
   /**
* Start time of the fact
*
* @return Start time of the fact {@link Date}
*/
-  public Date getStartTime();
+  Date getStartTime();
 
   /**
* Absolute end time of the fact
*
* @return Absolute End time of the fact {@link Date}
*/
-  public Date getAbsoluteEndTime();
+  Date getAbsoluteEndTime();
 
   /**
* Relative End time of the Fact
*
* @return Relative end time of the fact {@link Date}
*/
-  public Date getRelativeEndTime();
+  Date getRelativeEndTime();
 
   /**
* End time of the fact
*
* @return End time of the fact {@link Date}
*/
-  public Date getEndTime();
+  Date getEndTime();
 
   /**
* Is Virtual Fact
*
* @return true if fact is a virtual fact, false otherwise
*/
-  public boolean isVirtualFact();
+  boolean isVirtualFact();
 
   /**
* Storage name of the fact
*
* @return Storage name of the fact
*/
-  public String getSourceFactName();
+  String getSourceFactName();
 
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
--

[06/12] lens git commit: LENS-1437 : Missing unit test cases for virtual fact

2017-07-13 Thread prongs 
LENS-1437 : Missing unit test cases for virtual fact


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fbad3507
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fbad3507
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fbad3507

Branch: refs/heads/current-release-line
Commit: fbad350798a342a4de5e5b8b566a6f055338fde3
Parents: d49f45a
Author: Rajitha R 
Authored: Tue Jun 27 16:12:56 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:54 2017 +0530

--
 .../apache/lens/cube/parse/CubeTestSetup.java   |   4 +-
 .../lens/cube/parse/TestCubeRewriter.java   |  18 +--
 .../lens/cube/parse/TestVirtualFactQueries.java | 123 +++
 .../resources/schema/cubes/base/testcube2.xml   |  31 +
 .../resources/schema/cubes/base/virtualcube.xml |  23 +++-
 .../resources/schema/facts/testfact7_base.xml   |  67 ++
 .../resources/schema/facts/testfact8_base.xml   |  58 +
 .../resources/schema/facts/testfact9_base.xml   |  57 +
 .../schema/facts/virtual/virtualfact.xml|   2 +-
 9 files changed, 361 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 48869c2..0366e56 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -364,7 +364,7 @@ public class CubeTestSetup {
   // storageName[0] is hourly
   // storageName[1] is daily
   // storageName[2] is monthly
-  public static Map 
getWhereForMonthlyDailyAndHourly2months(String... storageTables) {
+  public static Map 
getWhereForMonthlyDailyAndHourly2months(String cubeName, String... 
storageTables) {
 Map storageTableToWhereClause = new LinkedHashMap();
 List hourlyparts = new ArrayList();
 List dailyparts = new ArrayList();
@@ -411,7 +411,7 @@ public class CubeTestSetup {
   tables.append(storageTables[0]);
 }
 Collections.sort(parts);
-storageTableToWhereClause.put(tables.toString(), 
StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
+storageTableToWhereClause.put(tables.toString(), 
StorageUtil.getWherePartClause("dt", cubeName, parts));
 return storageTableToWhereClause;
   }
 

http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index d8c7335..1c75e6c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -145,22 +145,6 @@ public class TestCubeRewriter extends TestQueryRewrite {
 //assertNotNull(rewrittenQuery.getNonExistingParts());
   }
 
-
-  @Test
-  public void testVirtualFactCubeSimpleQuery() throws Exception {
-Configuration conf = getConf();
-conf.set(DRIVER_SUPPORTED_STORAGES, "C1");
-CubeQueryContext rewrittenQuery =
-  rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, 
getConfWithStorages("C1"));
-String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select 
sum(virtualcube.msr2) as `sum(msr2)` FROM ",
-  null, "AND ( dim1 = 10 )", 
getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME,
-"C1_summary1"));
-String hql = rewrittenQuery.toHQL();
-compareQueries(hql, expected);
-System.out.println("Non existing parts:" + 
rewrittenQuery.getNonExistingParts());
-  }
-
-
   @Test
   public void testMaxCoveringFact() throws Exception {
 Configuration conf = getConf();
@@ -954,7 +938,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
   rewrite("select SUM(msr2) from testCube" + " where " + 
TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2"));
 String expected =
   getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as 
`sum(msr2)` FROM ", null, null,
-getWhereForMonthlyDailyAndHourly2months("C2_testfact"));
+getWhereForMonthlyDailyAndHourly2months(TEST_CUBE_NAME, 
"C2_testfact"));
 compareQueries(hqlQuery, expected);
   }

[11/12] lens git commit: LENS-1451: Enforcing valid fact tables doesn't filter out segmentations

2017-07-13 Thread prongs 
LENS-1451: Enforcing valid fact tables doesn't filter out segmentations


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3769ef0e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3769ef0e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3769ef0e

Branch: refs/heads/current-release-line
Commit: 3769ef0e8987caf26c931bf2735e3658e30f3ac4
Parents: 41051ea
Author: Rajat Khandelwal 
Authored: Thu Jul 6 19:35:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:43:00 2017 +0530

--
 .../lens/cube/parse/CandidateTableResolver.java | 26 ++--
 1 file changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/3769ef0e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index be3b474..86209bd 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -104,6 +104,9 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void populateCandidateTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
+  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
+  List validFactTables =
+StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
   List factTables = 
cubeql.getMetastoreClient().getAllFacts(cubeql.getCube());
   if (factTables.isEmpty()) {
 throw new 
LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(),
@@ -112,11 +115,15 @@ class CandidateTableResolver implements ContextRewriter {
   for (FactTable fact : factTables) {
 if (fact.getUpdatePeriods().isEmpty()) {
   log.info("Not considering fact: {} as it has no update periods", 
fact.getName());
+} else if (validFactTables != null && 
!validFactTables.contains(fact.getName())) {
+  log.info("Not considering fact: {} as it's not valid as per user 
configuration.", fact.getName());
 } else {
   for (String s : fact.getStorages()) {
 StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, 
s, cubeql);
 if (isStorageSupportedOnDriver(sc.getStorageName())) {
   cubeql.getCandidates().add(sc);
+} else {
+  log.info("Not considering {} since storage is not supported on 
driver.", sc.getName());
 }
   }
 }
@@ -125,7 +132,12 @@ class CandidateTableResolver implements ContextRewriter {
   log.info("Populated storage candidates: {}", cubeql.getCandidates());
   List segmentationCandidates = 
Lists.newArrayList();
   for (Segmentation segmentation : 
cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) {
-segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+if (validFactTables != null && 
!validFactTables.contains(segmentation.getName())) {
+  log.info("Not considering segmentation: {} as it's not valid as per 
user configuration.",
+segmentation.getName());
+} else {
+  segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+}
   }
   cubeql.getCandidates().addAll(segmentationCandidates);
 }
@@ -261,9 +273,6 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void resolveCandidateFactTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
-  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
-  List validFactTables =
-  StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
 
   Set queriedMsrs = new HashSet<>();
   Set dimExprs = new HashSet<>();
@@ -279,15 +288,6 @@ class CandidateTableResolver implements ContextRewriter {
 Candidate cand = i.next();
 if (cand instanceof StorageCandidate) {
   StorageCandidate sc = (StorageCandidate) cand;
-  if (validFactTables != null) {
-if 
(!validFactTables.contains(sc.getFact().getName().toLowerCase())) {
-  log.info("Not considering storage candidate:{} as it is not a 
valid candidate", sc);
-  cubeql.addStoragePruningMsg(sc, new 
CandidateTablePruneCause(CandidateTablePruneCode.INVALID));
-

[07/12] lens git commit: LENS-1448: Having clause expressions are not resolved correctly for JoinCandidates

2017-07-13 Thread prongs 
LENS-1448: Having clause expressions are not resolved correctly for 
JoinCandidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/54ab131a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/54ab131a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/54ab131a

Branch: refs/heads/current-release-line
Commit: 54ab131a2119a029dd8a470653d803fa0a7c35b6
Parents: fbad350
Author: Sushil Mohanty 
Authored: Wed Jun 28 13:10:55 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:55 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 11 +--
 .../lens/cube/parse/UnionQueryWriter.java   | 19 +++
 .../lens/cube/parse/TestBaseCubeQueries.java| 20 
 3 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 2403576..66b043e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -424,20 +424,19 @@ class ExpressionResolver implements ContextRewriter {
   }
   replaceAST(cubeql, queryAST.getJoinAST());
   replaceAST(cubeql, queryAST.getGroupByAST());
-  // Having AST is resolved by each fact, so that all facts can expand 
their expressions.
-  // Having ast is not copied now, it's maintained in cubeQueryContext, 
each fact processes that serially.
+  // Resolve having expression for StorageCandidate
   if (queryAST.getHavingAST() != null) {
 replaceAST(cubeql, queryAST.getHavingAST());
-  } else if (cubeql.getHavingAST() != null && 
nonPickedExpressionsForCandidate.isEmpty()) {
-replaceAST(cubeql, cubeql.getHavingAST());
-queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
+  } else if (cubeql.getHavingAST() != null) {
+ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+replaceAST(cubeql, havingCopy);
+queryAST.setHavingAST(havingCopy);
   }
   replaceAST(cubeql, queryAST.getOrderByAST());
 }
 
 private void replaceAST(final CubeQueryContext cubeql, ASTNode node) 
throws LensException {
   if (node == null) {
-
 return;
   }
   // Traverse the tree and resolve expression columns

http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f6c9ce1..9dc7ee6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -96,7 +96,6 @@ public class UnionQueryWriter extends SimpleHQLContext {
   if (sc.getQueryAst().getHavingAST() != null) {
 cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
   }
-  sc.getQueryAst().setHavingAST(null);
   sc.getQueryAst().setOrderByAST(null);
   sc.getQueryAst().setLimitValue(null);
 }
@@ -120,8 +119,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
   private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
-if (cubeql.getHavingAST() != null) {
-  ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+if (sc.getQueryAst().getHavingAST() == null
+&& cubeql.getHavingAST() != null) {
+  sc.getQueryAst().setHavingAST(cubeql.getHavingAST());
+}
+if (sc.getQueryAst().getHavingAST() != null) {
+  ASTNode havingCopy = 
MetastoreUtil.copyAST(sc.getQueryAst().getHavingAST());
   Set havingAggChildrenASTs = new LinkedHashSet<>();
   getAggregateChildrenInNode(havingCopy, havingAggChildrenASTs);
   processHavingExpression(innerAst, havingAggChildrenASTs, aliasDecider, 
sc);
@@ -283,8 +286,9 @@ public class UnionQueryWriter extends SimpleHQLContext {
   }
 
   private boolean isNodeDefault(ASTNode node) {
-if (HQLParser.isAggregateAST((ASTNode) node.getChild(0))) {
-  if (HQLParser.getString((ASTNode) 
node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)) {
+if (HQLParser.isAggregateAST((ASTNode) node.getChild(0)) ||