lens git commit: LENS-1432 : Map Join tuning broken for this query leading to Map timeouts

2017-06-06 Thread prongs 
Repository: lens
Updated Branches:
  refs/heads/master 359937fab -> 933cc3d6f


LENS-1432 : Map Join tuning broken for this query leading to Map timeouts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/933cc3d6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/933cc3d6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/933cc3d6

Branch: refs/heads/master
Commit: 933cc3d6f7924c893f1c18273ffba8347bfcec89
Parents: 359937f
Author: Sushil Mohanty 
Authored: Tue Jun 6 18:14:39 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 18:14:39 2017 +0530

--
 .../cube/parse/MultiCandidateQueryWriterContext.java | 11 +++
 .../org/apache/lens/cube/parse/QueryWriterContext.java   |  4 
 .../java/org/apache/lens/driver/cube/RewriterPlan.java   |  6 +++---
 .../org/apache/lens/cube/parse/TestRewriterPlan.java |  2 ++
 .../lens/cube/parse/TestUnionAndJoinCandidates.java  |  2 +-
 lens-cube/src/test/resources/schema/facts/b1fact1.xml|  2 +-
 6 files changed, 22 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
index d57c027..50046a5 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
@@ -18,8 +18,11 @@
  */
 package org.apache.lens.cube.parse;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.lens.cube.metadata.Dimension;
 import org.apache.lens.server.api.error.LensException;
 
 import com.google.common.collect.Lists;
@@ -90,4 +93,12 @@ public class MultiCandidateQueryWriterContext implements 
QueryWriterContext {
 List leafWriterContexts = 
getLeafQueryWriterContexts();
 return new UnionQueryWriter(leafWriterContexts, getCubeQueryContext());
   }
+
+  public Map getDimsToQuery() {
+Map allDimsQueried = new HashMap<>();
+for (QueryWriterContext ctx : children) {
+  allDimsQueried.putAll(ctx.getDimsToQuery());
+}
+return allDimsQueried;
+  }
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
index d55de1f..6531f22 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
@@ -18,6 +18,9 @@
  */
 package org.apache.lens.cube.parse;
 
+import java.util.Map;
+
+import org.apache.lens.cube.metadata.Dimension;
 import org.apache.lens.server.api.error.LensException;
 
 /**
@@ -31,4 +34,5 @@ public interface QueryWriterContext {
   QueryAST getQueryAst();
   void updateFromString() throws LensException;
   QueryWriter toQueryWriter() throws LensException;
+  Map getDimsToQuery();
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java 
b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
index caf8770..a27aec1 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -45,10 +45,10 @@ public final class RewriterPlan extends DriverQueryPlan {
 
   @SuppressWarnings("unchecked") // required for (Set) casting
   void extractPlan(Collection cubeQueries) {
-
 for (CubeQueryContext ctx : cubeQueries) {
-  if (ctx.getPickedDimTables() != null && 
!ctx.getPickedDimTables().isEmpty()) {
-for (CandidateDim dim : ctx.getPickedDimTables()) {
+  if (ctx.getQueryWriterContext().getDimsToQuery() != null
+  && !ctx.getQueryWriterContext().getDimsToQuery().isEmpty()) {
+for (CandidateDim dim : 
ctx.getQueryWriterContext().getDimsToQuery().values()) {
   addTablesQueried(dim.getStorageTable());
   if (partitions.get(dim.getName()) == null ||

lens git commit: LENS-1435: Cube xml files are missing in lens-example

2017-06-06 Thread prongs 
Repository: lens
Updated Branches:
  refs/heads/master 89b17e640 -> aaf455111


LENS-1435: Cube xml files are missing in lens-example


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/aaf45511
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/aaf45511
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/aaf45511

Branch: refs/heads/master
Commit: aaf455111953fd1d49218a8bdf4bcc9600b01899
Parents: 89b17e6
Author: Sushil Mohanty 
Authored: Tue Jun 6 17:45:22 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 17:45:22 2017 +0530

--
 lens-examples/src/main/resources/cube11.xml | 53 
 lens-examples/src/main/resources/cube22.xml | 53 
 lens-examples/src/main/resources/cube33.xml | 53 
 3 files changed, 159 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube11.xml
--
diff --git a/lens-examples/src/main/resources/cube11.xml 
b/lens-examples/src/main/resources/cube11.xml
new file mode 100644
index 000..2283939
--- /dev/null
+++ b/lens-examples/src/main/resources/cube11.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube22.xml
--
diff --git a/lens-examples/src/main/resources/cube22.xml 
b/lens-examples/src/main/resources/cube22.xml
new file mode 100644
index 000..59b1828
--- /dev/null
+++ b/lens-examples/src/main/resources/cube22.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube33.xml
--
diff --git a/lens-examples/src/main/resources/cube33.xml 
b/lens-examples/src/main/resources/cube33.xml
new file mode 100644
index 000..47b9b17
--- /dev/null
+++ b/lens-examples/src/main/resources/cube33.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

lens git commit: LENS-1430 : Expressions are not getting resolved correctly in union query

2017-06-06 Thread sushilmohanty 
Repository: lens
Updated Branches:
  refs/heads/current-release-line cc6c907d7 -> be75e1d55


LENS-1430 : Expressions are not getting resolved correctly in union query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/be75e1d5
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/be75e1d5
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/be75e1d5

Branch: refs/heads/current-release-line
Commit: be75e1d5571f33ca1ec51f91a5d113abe0838842
Parents: cc6c907
Author: Sushil Mohanty 
Authored: Tue Jun 6 16:43:17 2017 +0530
Committer: sushilmohanty 
Committed: Tue Jun 6 16:43:17 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 36 +++-
 .../cube/parse/TestUnionAndJoinCandidates.java  | 60 +++-
 .../resources/schema/cubes/base/basecube.xml| 12 
 .../resources/schema/cubes/base/testcube.xml|  9 +++
 .../cubes/derived/union_join_ctx_der1.xml   |  2 +
 .../schema/facts/union_join_ctx_fact1.xml   |  2 +-
 .../schema/facts/union_join_ctx_fact3.xml   |  1 +
 .../schema/facts/union_join_ctx_fact4.xml   | 59 +++
 8 files changed, 177 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/be75e1d5/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 7cad400..a2f4bb6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -285,6 +285,7 @@ class ExpressionResolver implements ContextRewriter {
 @Getter
 private Map allExprsQueried = new 
HashMap();
 private Map pickedExpressions = new 
HashMap();
+private Map nonPickedExpressionsForCandidate = new 
HashMap();
 private final CubeQueryContext cubeql;
 
 ExpressionResolverContext(CubeQueryContext cubeql) {
@@ -407,6 +408,7 @@ class ExpressionResolver implements ContextRewriter {
   }
 
   pickedExpressions.clear();
+  nonPickedExpressionsForCandidate.clear();
 
   return exprDims;
 }
@@ -426,7 +428,7 @@ class ExpressionResolver implements ContextRewriter {
   // Having ast is not copied now, it's maintained in cubeQueryContext, 
each fact processes that serially.
   if (queryAST.getHavingAST() != null) {
 replaceAST(cubeql, queryAST.getHavingAST());
-  } else if (cubeql.getHavingAST() != null) {
+  } else if (cubeql.getHavingAST() != null && 
nonPickedExpressionsForCandidate.isEmpty()) {
 replaceAST(cubeql, cubeql.getHavingAST());
 queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
   }
@@ -459,6 +461,9 @@ class ExpressionResolver implements ContextRewriter {
   if (expr != null) {
 node1.setChild(i, replaceAlias(expr.getRewrittenAST(), 
cubeql));
   }
+} else if (nonPickedExpressionsForCandidate.containsKey(column)) {
+  node1.setChild(i, nonPickedExpressionsForCandidate.get(column));
+
 }
   }
 }
@@ -477,7 +482,7 @@ class ExpressionResolver implements ContextRewriter {
   return null;
 }
 
-private void pickExpressionsForTable(CandidateTable cTable) {
+private void pickExpressionsForTable(CandidateTable cTable) throws 
LensException {
   for (Map.Entry ecEntry : 
allExprsQueried.entrySet()) {
 Set ecSet = ecEntry.getValue();
 for (ExpressionContext ec : ecSet) {
@@ -488,6 +493,8 @@ class ExpressionResolver implements ContextRewriter {
 // pick first evaluable expression
 pickedExpressions.computeIfAbsent(ecEntry.getKey(), k -> new 
HashSet<>())
   .add(new PickedExpression(ec.srcAlias, 
ec.evaluableExpressions.get(cTable).iterator().next()));
+  } else {
+nonPickedExpressionsForCandidate.put(ecEntry.getKey(), 
getDefaultExpr(getExprAst(ec)));
   }
 }
   }
@@ -495,6 +502,31 @@ class ExpressionResolver implements ContextRewriter {
   }
 }
 
+private ASTNode getExprAst(ExpressionContext ec) {
+  Set scSet = 
CandidateUtil.getStorageCandidates(cubeql.getCandidates());
+  Set storageTableNames = new HashSet();
+  for (StorageCandidate sc : scSet) {
+storageTableNames.add(sc.getStorageTable());
+  }
+  for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+

lens git commit: LENS-1427: ExpressionResolver is removing expressions if they are not answerable by *any* candidates

2017-06-06 Thread prongs 
Repository: lens
Updated Branches:
  refs/heads/master d3e732c9f -> cb2529672


LENS-1427: ExpressionResolver is removing expressions if they are not 
answerable by *any* candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cb252967
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cb252967
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cb252967

Branch: refs/heads/master
Commit: cb252967237a7e6c945a9031a3d417c18bc736cf
Parents: d3e732c
Author: Rajat Khandelwal 
Authored: Tue Jun 6 15:37:41 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 15:37:41 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 37 +---
 .../test/resources/schema/cubes/base/b1cube.xml |  2 +-
 .../test/resources/schema/cubes/base/b2cube.xml |  2 +-
 .../resources/schema/cubes/base/testcube.xml|  2 +-
 4 files changed, 19 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/cb252967/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 7cad400..f38aa54 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -518,15 +518,14 @@ class ExpressionResolver implements ContextRewriter {
 }
 // Remove expressions for which denormalized columns are no more 
reachable
 esc.getDeNormCtx().pruneReferences(cubeql);
-for (String table : 
esc.getDeNormCtx().getTableToRefCols().keySet()) {
-  Set nonReachableFields = 
esc.getDeNormCtx().getNonReachableReferenceFields(table);
-  if (!nonReachableFields.isEmpty()) {
-log.info("Removing expression {} as columns {} are not 
available", esc, nonReachableFields);
-iterator.remove();
-removedEsc.add(esc);
-removed = true;
-break;
-  }
+if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
+  && esc.getDeNormCtx().getTableToRefCols().keySet().stream()
+  
.map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty))
 {
+  log.info("Removing expression {} as all tables have non 
reachable fields", esc);
+  iterator.remove();
+  removedEsc.add(esc);
+  removed = true;
+  break;
 }
 if (removed) {
   continue;
@@ -630,19 +629,15 @@ class ExpressionResolver implements ContextRewriter {
 for (Map.Entry ecEntry : 
exprCtx.allExprsQueried.entrySet()) {
   String expr = ecEntry.getKey();
   Set ecSet = ecEntry.getValue();
-  for (ExpressionContext ec : ecSet) {
-if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) 
{
-  for (Iterator sItr = 
cubeql.getCandidates().iterator(); sItr.hasNext();) {
-Candidate cand = sItr.next();
-if (!cand.isExpressionEvaluable(ec)) {
-  log.info("Not considering Candidate :{} as {} is not 
evaluable", cand, ec.exprCol.getName());
-  sItr.remove();
-  cubeql.addCandidatePruningMsg(cand,
-  
CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
-}
-  }
+  cubeql.getCandidates().removeIf(x-> {
+if (ecSet.stream().noneMatch(x::isExpressionEvaluable)) {
+  log.info("Not considering Candidate :{} as {} is not evaluable", 
x, expr);
+  cubeql.addCandidatePruningMsg(x,
+CandidateTablePruneCause.expressionNotEvaluable(expr));
+  return true;
 }
-  }
+return false;
+  });
 }
   }
   // prune candidate dims without any valid expressions

http://git-wip-us.apache.org/repos/asf/lens/blob/cb252967/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
--
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml 
b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
index e3e9909..fd25982 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
@@ -178,7 +178,7 @@
   
 
 
-  
+

lens git commit: LENS-1429: Queries failing intermittently with NO_REF_COL_AVAILABLE

2017-06-06 Thread prongs 
Repository: lens
Updated Branches:
  refs/heads/master 159cf71e7 -> d3e732c9f


LENS-1429: Queries failing intermittently with NO_REF_COL_AVAILABLE


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d3e732c9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d3e732c9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d3e732c9

Branch: refs/heads/master
Commit: d3e732c9fe579b54d6c60e01a36746ecbd4ecb59
Parents: 159cf71
Author: Sushil Mohanty 
Authored: Tue Jun 6 15:35:46 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 15:35:46 2017 +0530

--
 .../org/apache/lens/cube/parse/CandidateTableResolver.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d3e732c9/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 93736cd..f530650 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -674,11 +674,13 @@ class CandidateTableResolver implements ContextRewriter {
 if (colSet == null || colSet.isEmpty()) {
   return true;
 }
+boolean isEvaluable = false;
 for (QueriedPhraseContext qur : colSet) {
   if (sc.isPhraseAnswerable(qur)) {
-return true;
+isEvaluable = true;
+continue;
   }
 }
-return false;
+return isEvaluable;
   }
 }

lens git commit: LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate

2017-06-06 Thread sushilmohanty 
Repository: lens
Updated Branches:
  refs/heads/master 5de45e0f8 -> 159cf71e7


LENS-1428 : Common measures are getting repeated in UnionQuery for JoinCandidate


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/159cf71e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/159cf71e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/159cf71e

Branch: refs/heads/master
Commit: 159cf71e732a0b63bf8c7efde6489f4eea8d943a
Parents: 5de45e0
Author: Sushil Mohanty 
Authored: Tue Jun 6 15:08:38 2017 +0530
Committer: sushilmohanty 
Committed: Tue Jun 6 15:08:38 2017 +0530

--
 .../apache/lens/cube/metadata/CubeMetastoreClient.java |  2 +-
 .../java/org/apache/lens/cube/parse/Candidate.java |  9 +
 .../org/apache/lens/cube/parse/CubeQueryContext.java   |  5 +
 .../java/org/apache/lens/cube/parse/JoinCandidate.java | 13 +
 .../org/apache/lens/cube/parse/StorageCandidate.java   |  7 +++
 .../org/apache/lens/cube/parse/UnionQueryWriter.java   |  6 ++
 6 files changed, 37 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 749e44c..5c05e53 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -443,7 +443,7 @@ public class CubeMetastoreClient {
 if (get(timeLineKey) == null) {
   loadTimeLines(fact, storage, timeLineKey);
 }
-log.info("timeline for {} is: {}", storage, get(timeLineKey));
+log.debug("timeline for {} is: {}", storage, get(timeLineKey));
 // return the final value from memory
 return get(timeLineKey);
 // RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck

http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
--
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
index 2edeb41..0855ced 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java
@@ -355,4 +355,13 @@ public interface Candidate {
 }
 throw new IllegalArgumentException("Candidate doesn't have children and no 
suitable implementation found");
   }
+
+  default Set decideMeasurePhrasesToAnswer(Set 
measurePhraseIndices) throws LensException {
+HashSet allCovered = Sets.newHashSet();
+for (Candidate candidate : getChildren()) {
+  Set covered = 
candidate.decideMeasurePhrasesToAnswer(measurePhraseIndices);
+  allCovered.addAll(covered);
+}
+return allCovered;
+  }
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 47bce65..b5330a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -830,6 +830,11 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST,
 cand = iter.next();
 log.info("Available Candidates:{}, picking up Candidate: {} for 
querying", candidates, cand);
 pickedCandidate = cand;
+// Answerable common measures in JoinCandidate should be answered by 
one of the children, otherwise
+// measure numbers will be added multiple times in the final union 
query.
+Set measureIndices = getQueriedPhrases().stream().filter(x -> 
x.hasMeasures(this))
+  .map(QueriedPhraseContext::getPosition).collect(toSet());
+pickedCandidate.decideMeasurePhrasesToAnswer(measureIndices);
   }
   if (pickedCandidate == null) {
 throwNoCandidateFactException();

http://git-wip-us.apache.org/repos/asf/lens/blob/159cf71e/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
--
diff --git

[3/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
LENS-1412 : Add capability to define virtual facts in a cube


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/27ad9aa8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/27ad9aa8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/27ad9aa8

Branch: refs/heads/current-release-line
Commit: 27ad9aa8f27082a8ac6d82cbf91fff80b43ab689
Parents: 7c1171c
Author: Rajitha R 
Authored: Tue Jun 6 13:59:58 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Tue Jun 6 14:01:49 2017 +0530

--
 .../lens/api/metastore/SchemaTraverser.java |   1 +
 lens-api/src/main/resources/cube-0.1.xsd| 176 +++
 lens-api/src/main/resources/lens-errors.conf|   6 +
 .../lens/cli/commands/LensFactCommands.java |   6 +-
 .../lens/cli/commands/LensSchemaCommands.java   |  18 +-
 .../apache/lens/cli/TestLensFactCommands.java   | 132 +-
 .../apache/lens/cli/TestLensSchemaCommands.java |   2 +-
 .../schema/cubes/base/virtual-cube.xml  |  30 ++
 .../resources/schema/facts/virtual_fact.xml |  28 ++
 .../java/org/apache/lens/client/LensClient.java |   2 +-
 .../apache/lens/client/LensMetadataClient.java  |  18 +-
 .../lens/cube/error/LensCubeErrorCode.java  |   3 +-
 .../lens/cube/metadata/AbstractCubeTable.java   |  22 +-
 .../lens/cube/metadata/CubeFactTable.java   |  52 ++-
 .../lens/cube/metadata/CubeMetastoreClient.java | 463 ++-
 .../cube/metadata/CubeVirtualFactTable.java | 186 
 .../apache/lens/cube/metadata/FactTable.java| 166 +++
 .../apache/lens/cube/metadata/JAXBUtils.java|  24 +-
 .../lens/cube/metadata/MetastoreConstants.java  |   2 +
 .../lens/cube/metadata/MetastoreUtil.java   |  26 +-
 .../apache/lens/cube/metadata/Segmentation.java |  13 +-
 .../org/apache/lens/cube/parse/Candidate.java   |   8 +-
 .../lens/cube/parse/CandidateTableResolver.java |  11 +-
 .../lens/cube/parse/StorageCandidate.java   |  55 +--
 .../cube/parse/StorageCandidateHQLContext.java  |  12 +-
 .../lens/cube/parse/StorageTableResolver.java   |   5 +-
 .../lens/cube/metadata/CubeFactTableTest.java   |  12 +-
 .../cube/metadata/TestCubeMetastoreClient.java  | 123 -
 .../apache/lens/cube/parse/CubeTestSetup.java   |  40 +-
 .../lens/cube/parse/TestCubeRewriter.java   |  16 +
 .../parse/TestCubeSegmentationRewriter.java |  15 +-
 .../lens/cube/parse/TestUnionQueries.java   |   2 +-
 .../resources/schema/cubes/base/virtualcube.xml |  36 ++
 .../test/resources/schema/facts/virtualfact.xml |  27 ++
 lens-driver-es/pom.xml  |   3 +-
 lens-examples/pom.xml   |   3 +-
 .../src/test/resources/yaml/fact1.yaml  |   4 +-
 .../src/test/resources/yaml/fact2.yaml  |   4 +-
 .../src/test/resources/yaml/rawfact.yaml|   4 +-
 .../yaml/sales-aggr-continuous-fact.yaml|   4 +-
 .../test/resources/yaml/sales-aggr-fact1.yaml   |   4 +-
 .../test/resources/yaml/sales-aggr-fact2.yaml   |   4 +-
 .../src/test/resources/yaml/sales-raw-fact.yaml |   4 +-
 .../api/metastore/CubeMetastoreService.java |   8 +-
 lens-server/pom.xml |   2 +-
 .../metastore/CubeMetastoreServiceImpl.java |  24 +-
 .../server/metastore/MetastoreResource.java |  17 +-
 .../lens/server/common/RestAPITestUtil.java |   7 +-
 .../server/metastore/TestMetastoreService.java  | 274 ++-
 src/site/apt/user/cli.apt   |   3 +-
 tools/scripts/generate-site-public.sh   |   2 +-
 51 files changed, 1646 insertions(+), 463 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/27ad9aa8/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
--
diff --git 
a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java 
b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
index 9564443..09c848a 100644
--- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
+++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
@@ -42,6 +42,7 @@ public class SchemaTraverser implements Runnable {
 types.put("cubes/derived", XDerivedCube.class);
 types.put("dimensions", XDimension.class);
 types.put("facts", XFactTable.class);
+types.put("facts/virtual", XVirtualFactTable.class);
 types.put("dimtables", XDimensionTable.class);
 types.put("dimensiontables", XDimensionTable.class);
 types.put("dimensiontables", XDimensionTable.class);

http://git-wip-us.apache.org/repos/asf/lens/blob/27ad9aa8/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git

[1/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
Repository: lens
Updated Branches:
  refs/heads/current-release-line 7c1171c8d -> 27ad9aa8f


http://git-wip-us.apache.org/repos/asf/lens/blob/27ad9aa8/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 6d32d23..ab24033 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -1856,6 +1856,24 @@ public class TestMetastoreService extends LensJerseyTest 
{
 return f;
   }
 
+
+  private XVirtualFactTable createVirtualFactTable(String factName, String 
sourceFactName) {
+return createVirtualFactTable(factName, "testCube", sourceFactName);
+  }
+
+  private XVirtualFactTable createVirtualFactTable(String factName, final 
String cubeName, String sourceFactName) {
+XVirtualFactTable f = cubeObjectFactory.createXVirtualFactTable();
+f.setProperties(new XProperties());
+f.setName(factName);
+f.setWeight(10.0);
+f.setCubeName(cubeName);
+f.setSourceFactName(sourceFactName);
+
+Map properties = LensUtil.getHashMap("foo1", "bar1");
+
f.getProperties().getProperty().addAll(JAXBUtils.xPropertiesFromMap(properties));
+return f;
+  }
+
   @Test(dataProvider = "mediaTypeData")
   public void testCreateFactTableWithMultipleUpdatePeriods(MediaType 
mediaType) throws Exception {
 
@@ -1877,7 +1895,7 @@ public class TestMetastoreService extends LensJerseyTest {
 .add(createStorageTblWithMultipleTableDescriptors("S1", tables, 
updatePeriods));
   APIResult result = 
target().path("metastore").path("facts").queryParam("sessionid", lensSessionId)
 .request(mediaType)
-.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFactTable(f)) {
+.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFact(f)) {
 }, mediaType), APIResult.class);
   assertSuccess(result);
 
@@ -1886,10 +1904,10 @@ public class TestMetastoreService extends 
LensJerseyTest {
   assertTrue(factNames.getElements().contains(table.toLowerCase()));
 
   // Get the created tables
-  JAXBElement gotFactElement = 
target().path("metastore/facts").path(table)
-.queryParam("sessionid", lensSessionId).request(mediaType).get(new 
GenericType() {
+  JAXBElement gotFactElement = 
target().path("metastore/facts").path(table)
+.queryParam("sessionid", lensSessionId).request(mediaType).get(new 
GenericType() {
 });
-  XFactTable gotFact = gotFactElement.getValue();
+  XFactTable gotFact = (XFactTable) gotFactElement.getValue();
   assertTrue(gotFact.getName().equalsIgnoreCase(table));
   assertEquals(gotFact.getWeight(), 10.0);
 
@@ -1915,15 +1933,15 @@ public class TestMetastoreService extends 
LensJerseyTest {
   // Update
   result = 
target().path("metastore").path("facts").path(table).queryParam("sessionid", 
lensSessionId)
 .request(mediaType)
-.put(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFactTable(update))
 {
+.put(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFact(update)) {
 }, mediaType), APIResult.class);
   assertSuccess(result);
 
   // Get the updated table
   gotFactElement = 
target().path("metastore/facts").path(table).queryParam("sessionid", 
lensSessionId)
-.request(mediaType).get(new GenericType() {
+.request(mediaType).get(new GenericType() {
 });
-  gotFact = gotFactElement.getValue();
+  gotFact = (XFactTable) gotFactElement.getValue();
   CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact);
   assertEquals(ucf.weight(), 20.0);
   assertTrue(ucf.getUpdatePeriods().get("S1").contains(HOURLY));
@@ -2032,6 +2050,7 @@ public class TestMetastoreService extends LensJerseyTest {
 
   @Test(dataProvider = "mediaTypeData")
   public void testCreateFactTable(MediaType mediaType) throws Exception {
+
 final String table = "testCreateFactTable";
 final String DB = dbPFX + "testCreateFactTable_DB" + 
mediaType.getSubtype();
 String prevDb = getCurrentDatabase(mediaType);
@@ -2049,7 +2068,7 @@ public class TestMetastoreService extends LensJerseyTest {
 .path("facts").queryParam("sessionid", lensSessionId)
 .request(mediaType)
 .post(Entity.entity(
-  new 
GenericEntity(cubeObjectFactory.createXFactTable(f)){},
 mediaType),
+  new 
GenericEntity(cubeObjectFactory.createXFact(f)){}, 
mediaType),
   APIResult.class);

[2/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
http://git-wip-us.apache.org/repos/asf/lens/blob/27ad9aa8/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
new file mode 100644
index 000..f87cf44
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+
+/**
+ * The Fact table interface
+ */
+public interface FactTable extends Named {
+
+  /**
+   * Get map of storage to update period mapping
+   *
+   * @return Map of storage to set of update periods
+   */
+  public Map getUpdatePeriods();
+
+  /**
+   * Cube to which this fact belongs to
+   *
+   * @return the cube string
+   */
+  public String getCubeName();
+
+  /**
+   * The set of Storage names
+   *
+   * @return set of strings
+   */
+  public Set getStorages();
+
+  /**
+   *The type of the fact
+   *
+   * @return table type {@link CubeTableType}
+   */
+  public CubeTableType getTableType();
+
+  /**
+   * Config properties
+   *
+   * @return map of string, string
+   */
+  public Map getProperties();
+
+  /**
+   * Valid columns of the fact
+   *
+   * @return list of column names
+   */
+  public Set getValidColumns();
+
+  /**
+   * Weight of the fact
+   *
+   * @return weight of the fact in double
+   */
+  public double weight();
+
+  /**
+   * Set of all the columns names of the fact
+   *
+   * @return set of column names
+   */
+  public Set getAllFieldNames();
+
+  /**
+   *tag for checking data completeness
+   *
+   * @return Tag String
+   */
+  public String getDataCompletenessTag();
+
+  /**
+   * List of columns of the fact
+   *
+   * @return set of {@link FieldSchema}
+   */
+  public List getColumns();
+
+  /**
+   * Is Aggregated Fact
+   *
+   * @return true if fact is Aggregated , false otherwise
+   */
+  public boolean isAggregated();
+
+  /**
+   * Absolute start time of the fact
+   *
+   * @return Absolute Start time of the fact {@link Date}
+   */
+  public Date getAbsoluteStartTime();
+
+  /**
+   * Relative start time of the fact
+   *
+   * @return Relative Start time of the fact {@link Date}
+   */
+  public Date getRelativeStartTime();
+
+  /**
+   * Start time of the fact
+   *
+   * @return Start time of the fact {@link Date}
+   */
+  public Date getStartTime();
+
+  /**
+   * Absolute end time of the fact
+   *
+   * @return Absolute End time of the fact {@link Date}
+   */
+  public Date getAbsoluteEndTime();
+
+  /**
+   * Relative End time of the Fact
+   *
+   * @return Relative end time of the fact {@link Date}
+   */
+  public Date getRelativeEndTime();
+
+  /**
+   * End time of the fact
+   *
+   * @return End time of the fact {@link Date}
+   */
+  public Date getEndTime();
+
+  /**
+   * Is Virtual Fact
+   *
+   * @return true if fact is a virtual fact, false otherwise
+   */
+  public boolean isVirtualFact();
+
+  /**
+   * Storage name of the fact
+   *
+   * @return Storage name of the fact
+   */
+  public String getSourceFactName();
+
+}

http://git-wip-us.apache.org/repos/asf/lens/blob/27ad9aa8/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
index e1e3d16..7dfb166 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
@@ -718,6 +718,15 @@ public final class JAXBUtils {
   mapFromXProperties(fact.getProperties()), storageTablePrefixMap);
   }
 
+  public static CubeVirtualFactTable 
cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable)
+

[1/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
Repository: lens
Updated Branches:
  refs/heads/master 34500f1b3 -> 5de45e0f8


http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 6d32d23..ab24033 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -1856,6 +1856,24 @@ public class TestMetastoreService extends LensJerseyTest 
{
 return f;
   }
 
+
+  private XVirtualFactTable createVirtualFactTable(String factName, String 
sourceFactName) {
+return createVirtualFactTable(factName, "testCube", sourceFactName);
+  }
+
+  private XVirtualFactTable createVirtualFactTable(String factName, final 
String cubeName, String sourceFactName) {
+XVirtualFactTable f = cubeObjectFactory.createXVirtualFactTable();
+f.setProperties(new XProperties());
+f.setName(factName);
+f.setWeight(10.0);
+f.setCubeName(cubeName);
+f.setSourceFactName(sourceFactName);
+
+Map properties = LensUtil.getHashMap("foo1", "bar1");
+
f.getProperties().getProperty().addAll(JAXBUtils.xPropertiesFromMap(properties));
+return f;
+  }
+
   @Test(dataProvider = "mediaTypeData")
   public void testCreateFactTableWithMultipleUpdatePeriods(MediaType 
mediaType) throws Exception {
 
@@ -1877,7 +1895,7 @@ public class TestMetastoreService extends LensJerseyTest {
 .add(createStorageTblWithMultipleTableDescriptors("S1", tables, 
updatePeriods));
   APIResult result = 
target().path("metastore").path("facts").queryParam("sessionid", lensSessionId)
 .request(mediaType)
-.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFactTable(f)) {
+.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFact(f)) {
 }, mediaType), APIResult.class);
   assertSuccess(result);
 
@@ -1886,10 +1904,10 @@ public class TestMetastoreService extends 
LensJerseyTest {
   assertTrue(factNames.getElements().contains(table.toLowerCase()));
 
   // Get the created tables
-  JAXBElement gotFactElement = 
target().path("metastore/facts").path(table)
-.queryParam("sessionid", lensSessionId).request(mediaType).get(new 
GenericType() {
+  JAXBElement gotFactElement = 
target().path("metastore/facts").path(table)
+.queryParam("sessionid", lensSessionId).request(mediaType).get(new 
GenericType() {
 });
-  XFactTable gotFact = gotFactElement.getValue();
+  XFactTable gotFact = (XFactTable) gotFactElement.getValue();
   assertTrue(gotFact.getName().equalsIgnoreCase(table));
   assertEquals(gotFact.getWeight(), 10.0);
 
@@ -1915,15 +1933,15 @@ public class TestMetastoreService extends 
LensJerseyTest {
   // Update
   result = 
target().path("metastore").path("facts").path(table).queryParam("sessionid", 
lensSessionId)
 .request(mediaType)
-.put(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFactTable(update))
 {
+.put(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFact(update)) {
 }, mediaType), APIResult.class);
   assertSuccess(result);
 
   // Get the updated table
   gotFactElement = 
target().path("metastore/facts").path(table).queryParam("sessionid", 
lensSessionId)
-.request(mediaType).get(new GenericType() {
+.request(mediaType).get(new GenericType() {
 });
-  gotFact = gotFactElement.getValue();
+  gotFact = (XFactTable) gotFactElement.getValue();
   CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact);
   assertEquals(ucf.weight(), 20.0);
   assertTrue(ucf.getUpdatePeriods().get("S1").contains(HOURLY));
@@ -2032,6 +2050,7 @@ public class TestMetastoreService extends LensJerseyTest {
 
   @Test(dataProvider = "mediaTypeData")
   public void testCreateFactTable(MediaType mediaType) throws Exception {
+
 final String table = "testCreateFactTable";
 final String DB = dbPFX + "testCreateFactTable_DB" + 
mediaType.getSubtype();
 String prevDb = getCurrentDatabase(mediaType);
@@ -2049,7 +2068,7 @@ public class TestMetastoreService extends LensJerseyTest {
 .path("facts").queryParam("sessionid", lensSessionId)
 .request(mediaType)
 .post(Entity.entity(
-  new 
GenericEntity(cubeObjectFactory.createXFactTable(f)){},
 mediaType),
+  new 
GenericEntity(cubeObjectFactory.createXFact(f)){}, 
mediaType),
   APIResult.class);
   assertSuccess(result);
 
@@

[3/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
LENS-1412 : Add capability to define virtual facts in a cube


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5de45e0f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5de45e0f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5de45e0f

Branch: refs/heads/master
Commit: 5de45e0f8212c2e03eaf9886b720e13868a1c42a
Parents: 34500f1
Author: Rajitha R 
Authored: Tue Jun 6 13:59:58 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Tue Jun 6 13:59:58 2017 +0530

--
 .../lens/api/metastore/SchemaTraverser.java |   1 +
 lens-api/src/main/resources/cube-0.1.xsd| 176 +++
 lens-api/src/main/resources/lens-errors.conf|   6 +
 .../lens/cli/commands/LensFactCommands.java |   6 +-
 .../lens/cli/commands/LensSchemaCommands.java   |  18 +-
 .../apache/lens/cli/TestLensFactCommands.java   | 132 +-
 .../apache/lens/cli/TestLensSchemaCommands.java |   2 +-
 .../schema/cubes/base/virtual-cube.xml  |  30 ++
 .../resources/schema/facts/virtual_fact.xml |  28 ++
 .../java/org/apache/lens/client/LensClient.java |   2 +-
 .../apache/lens/client/LensMetadataClient.java  |  18 +-
 .../lens/cube/error/LensCubeErrorCode.java  |   3 +-
 .../lens/cube/metadata/AbstractCubeTable.java   |  22 +-
 .../lens/cube/metadata/CubeFactTable.java   |  52 ++-
 .../lens/cube/metadata/CubeMetastoreClient.java | 463 ++-
 .../cube/metadata/CubeVirtualFactTable.java | 186 
 .../apache/lens/cube/metadata/FactTable.java| 166 +++
 .../apache/lens/cube/metadata/JAXBUtils.java|  24 +-
 .../lens/cube/metadata/MetastoreConstants.java  |   2 +
 .../lens/cube/metadata/MetastoreUtil.java   |  26 +-
 .../apache/lens/cube/metadata/Segmentation.java |  13 +-
 .../org/apache/lens/cube/parse/Candidate.java   |   8 +-
 .../lens/cube/parse/CandidateTableResolver.java |  11 +-
 .../lens/cube/parse/StorageCandidate.java   |  55 +--
 .../cube/parse/StorageCandidateHQLContext.java  |  12 +-
 .../lens/cube/parse/StorageTableResolver.java   |   5 +-
 .../lens/cube/metadata/CubeFactTableTest.java   |  12 +-
 .../cube/metadata/TestCubeMetastoreClient.java  | 123 -
 .../apache/lens/cube/parse/CubeTestSetup.java   |  40 +-
 .../lens/cube/parse/TestCubeRewriter.java   |  16 +
 .../parse/TestCubeSegmentationRewriter.java |  15 +-
 .../lens/cube/parse/TestUnionQueries.java   |   2 +-
 .../resources/schema/cubes/base/virtualcube.xml |  36 ++
 .../test/resources/schema/facts/virtualfact.xml |  27 ++
 lens-driver-es/pom.xml  |   3 +-
 lens-examples/pom.xml   |   3 +-
 .../src/test/resources/yaml/fact1.yaml  |   4 +-
 .../src/test/resources/yaml/fact2.yaml  |   4 +-
 .../src/test/resources/yaml/rawfact.yaml|   4 +-
 .../yaml/sales-aggr-continuous-fact.yaml|   4 +-
 .../test/resources/yaml/sales-aggr-fact1.yaml   |   4 +-
 .../test/resources/yaml/sales-aggr-fact2.yaml   |   4 +-
 .../src/test/resources/yaml/sales-raw-fact.yaml |   4 +-
 .../api/metastore/CubeMetastoreService.java |   8 +-
 lens-server/pom.xml |   2 +-
 .../metastore/CubeMetastoreServiceImpl.java |  24 +-
 .../server/metastore/MetastoreResource.java |  17 +-
 .../lens/server/common/RestAPITestUtil.java |   7 +-
 .../server/metastore/TestMetastoreService.java  | 274 ++-
 src/site/apt/user/cli.apt   |   3 +-
 tools/scripts/generate-site-public.sh   |   2 +-
 51 files changed, 1646 insertions(+), 463 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
--
diff --git 
a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java 
b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
index 9564443..09c848a 100644
--- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
+++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
@@ -42,6 +42,7 @@ public class SchemaTraverser implements Runnable {
 types.put("cubes/derived", XDerivedCube.class);
 types.put("dimensions", XDimension.class);
 types.put("facts", XFactTable.class);
+types.put("facts/virtual", XVirtualFactTable.class);
 types.put("dimtables", XDimensionTable.class);
 types.put("dimensiontables", XDimensionTable.class);
 types.put("dimensiontables", XDimensionTable.class);

http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git

[2/3] lens git commit: LENS-1412 : Add capability to define virtual facts in a cube

2017-06-06 Thread amareshwari 
http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
new file mode 100644
index 000..f87cf44
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+
+/**
+ * The Fact table interface
+ */
+public interface FactTable extends Named {
+
+  /**
+   * Get map of storage to update period mapping
+   *
+   * @return Map of storage to set of update periods
+   */
+  public Map getUpdatePeriods();
+
+  /**
+   * Cube to which this fact belongs to
+   *
+   * @return the cube string
+   */
+  public String getCubeName();
+
+  /**
+   * The set of Storage names
+   *
+   * @return set of strings
+   */
+  public Set getStorages();
+
+  /**
+   *The type of the fact
+   *
+   * @return table type {@link CubeTableType}
+   */
+  public CubeTableType getTableType();
+
+  /**
+   * Config properties
+   *
+   * @return map of string, string
+   */
+  public Map getProperties();
+
+  /**
+   * Valid columns of the fact
+   *
+   * @return list of column names
+   */
+  public Set getValidColumns();
+
+  /**
+   * Weight of the fact
+   *
+   * @return weight of the fact in double
+   */
+  public double weight();
+
+  /**
+   * Set of all the columns names of the fact
+   *
+   * @return set of column names
+   */
+  public Set getAllFieldNames();
+
+  /**
+   *tag for checking data completeness
+   *
+   * @return Tag String
+   */
+  public String getDataCompletenessTag();
+
+  /**
+   * List of columns of the fact
+   *
+   * @return set of {@link FieldSchema}
+   */
+  public List getColumns();
+
+  /**
+   * Is Aggregated Fact
+   *
+   * @return true if fact is Aggregated , false otherwise
+   */
+  public boolean isAggregated();
+
+  /**
+   * Absolute start time of the fact
+   *
+   * @return Absolute Start time of the fact {@link Date}
+   */
+  public Date getAbsoluteStartTime();
+
+  /**
+   * Relative start time of the fact
+   *
+   * @return Relative Start time of the fact {@link Date}
+   */
+  public Date getRelativeStartTime();
+
+  /**
+   * Start time of the fact
+   *
+   * @return Start time of the fact {@link Date}
+   */
+  public Date getStartTime();
+
+  /**
+   * Absolute end time of the fact
+   *
+   * @return Absolute End time of the fact {@link Date}
+   */
+  public Date getAbsoluteEndTime();
+
+  /**
+   * Relative End time of the Fact
+   *
+   * @return Relative end time of the fact {@link Date}
+   */
+  public Date getRelativeEndTime();
+
+  /**
+   * End time of the fact
+   *
+   * @return End time of the fact {@link Date}
+   */
+  public Date getEndTime();
+
+  /**
+   * Is Virtual Fact
+   *
+   * @return true if fact is a virtual fact, false otherwise
+   */
+  public boolean isVirtualFact();
+
+  /**
+   * Storage name of the fact
+   *
+   * @return Storage name of the fact
+   */
+  public String getSourceFactName();
+
+}

http://git-wip-us.apache.org/repos/asf/lens/blob/5de45e0f/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
index e1e3d16..7dfb166 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
@@ -718,6 +718,15 @@ public final class JAXBUtils {
   mapFromXProperties(fact.getProperties()), storageTablePrefixMap);
   }
 
+  public static CubeVirtualFactTable 
cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable)
+