lens git commit: LENS-1469: Support drop partition(s) for specific update periods

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master a86cb883f -> cdac40874


LENS-1469: Support drop partition(s) for specific update periods


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cdac4087
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cdac4087
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cdac4087

Branch: refs/heads/master
Commit: cdac40874c09cb80dd17a70908c86c3ac5922ac7
Parents: a86cb88
Author: Amit Khanna 
Authored: Wed Oct 25 13:05:29 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Oct 25 13:05:29 2017 +0530

--
 .../commands/LensDimensionTableCommands.java|  5 ++
 .../lens/cli/commands/LensFactCommands.java | 15 +++-
 .../cli/commands/LogicalTableCrudCommand.java   |  7 ++
 .../apache/lens/cli/TestLensFactCommands.java   |  2 +-
 .../java/org/apache/lens/client/LensClient.java |  4 +
 .../apache/lens/client/LensMetadataClient.java  |  7 +-
 .../api/metastore/CubeMetastoreService.java |  3 +
 .../metastore/CubeMetastoreServiceImpl.java | 10 ++-
 .../server/metastore/MetastoreResource.java |  5 +-
 .../server/metastore/TestMetastoreService.java  | 66 +-
 src/site/apt/user/cli.apt   | 92 +++-
 11 files changed, 160 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/cdac4087/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
--
diff --git 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
index 26650c1..f37a380 100644
--- 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
+++ 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
@@ -322,6 +322,11 @@ public class LensDimensionTableCommands extends 
LogicalTableCrudCommandhttp://git-wip-us.apache.org/repos/asf/lens/blob/cdac4087/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
--
diff --git 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index a872998..63dd992 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -198,9 +198,10 @@ public class LensFactCommands extends 
LogicalTableCrudCommand {
   /**
* Drop all partitions of fact.
*
-   * @param tableName   fact name
-   * @param storageName storage name
-   * @param filter  partition query filter
+   * @param tableName   fact name
+   * @param storageName storage name
+   * @param filter  partition query filter
+   * @param updatePeriodupdate period of partition to be dropped
* @return the string
*/
   @CliCommand(value = "fact drop partitions",
@@ -209,8 +210,9 @@ public class LensFactCommands extends 
LogicalTableCrudCommand {
   public String dropAllPartitionsOfFact(
 @CliOption(key = {"", "fact_name"}, mandatory = true, help = 
"") String tableName,
 @CliOption(key = {"", "storage_name"}, mandatory = true, help = 
"") String storageName,
+@CliOption(key = {"", "update_period"}, mandatory = true, help = 
"") String updatePeriod,
 @CliOption(key = {"", "filter"}, mandatory = false, help = 
"") String filter) {
-return dropPartitions(tableName, storageName, filter);
+return dropPartitions(tableName, storageName, filter, updatePeriod);
   }
 
   /**
@@ -327,6 +329,11 @@ public class LensFactCommands extends 
LogicalTableCrudCommand {
   }
 
   @Override
+  protected APIResult doDropPartitions(String tableName, String storageName, 
String filter, String updatePeriod) {
+return getClient().dropAllPartitionsOfFact(tableName, storageName, filter, 
updatePeriod);
+  }
+
+  @Override
   protected APIResult doUpdatePartition(String tableName, String storageName, 
String validPath) {
 return getClient().updatePartitionOfFact(tableName, storageName, 
validPath);
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/cdac4087/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
--
diff --git 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
index e0b60c2..5b0c1d0 100644
--- 

lens git commit: LENS-1473: Cubevirtualfact table relative start time has incorrect config field name

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 2b86ac2b8 -> 7c9707148


LENS-1473: Cubevirtualfact table relative start time has incorrect config field 
name


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7c970714
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7c970714
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7c970714

Branch: refs/heads/master
Commit: 7c97071481e6efc0fcd22ee00887d5c365ef497c
Parents: 2b86ac2
Author: Rajitha R 
Authored: Wed Sep 6 18:33:38 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Sep 6 18:33:38 2017 +0530

--
 .../java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/7c970714/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
index ce2a1a6..8b55985 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -147,7 +147,7 @@ public class CubeVirtualFactTable extends AbstractCubeTable 
implements FactTable
   }
 
   public Date getRelativeStartTime() {
-String relativeStartTime = 
this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME);
+String relativeStartTime = 
this.getProperties().get(MetastoreConstants.FACT_RELATIVE_START_TIME);
 Date relativeDate = null;
 if (StringUtils.isNotBlank(relativeStartTime)) {
   relativeDate = MetastoreUtil.getDateFromProperty(relativeStartTime, 
true, true);

lens git commit: LENS-1472: Populate sample metastore on example db is failing

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master caf7a2c0b -> 2b86ac2b8


LENS-1472: Populate sample metastore on example db is failing


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2b86ac2b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2b86ac2b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2b86ac2b

Branch: refs/heads/master
Commit: 2b86ac2b83608efb95d9c89c10445c4a30f5
Parents: caf7a2c
Author: Rajat Khandelwal 
Authored: Tue Sep 5 22:49:00 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Sep 5 22:49:00 2017 +0530

--
 lens-dist/src/main/assembly/bin-dist.xml | 8 
 .../org/apache/lens/examples/PopulateSampleMetastore.java| 5 +
 .../main/java/org/apache/lens/examples/SampleMetastore.java  | 3 +++
 .../src/main/resources/sales-aggr-fact2-local4.data  | 2 ++
 4 files changed, 18 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/2b86ac2b/lens-dist/src/main/assembly/bin-dist.xml
--
diff --git a/lens-dist/src/main/assembly/bin-dist.xml 
b/lens-dist/src/main/assembly/bin-dist.xml
index c3f8f17..36783b1 100644
--- a/lens-dist/src/main/assembly/bin-dist.xml
+++ b/lens-dist/src/main/assembly/bin-dist.xml
@@ -331,6 +331,14 @@
 
 
   ../lens-examples/src/main/resources/
+  
/client/examples/data/sales_aggr_fact2_local4
+  
+sales-aggr-fact2-local4*.data
+  
+
+
+
+  ../lens-examples/src/main/resources/
   /client/examples/data/sales_raw_local1
   
 sales-raw-local1*.data

http://git-wip-us.apache.org/repos/asf/lens/blob/2b86ac2b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
--
diff --git 
a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
 
b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
index 4fc15a6..7c8fd2c 100644
--- 
a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
+++ 
b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
@@ -43,6 +43,10 @@ public class PopulateSampleMetastore {
   private static final SimpleDateFormat FORMAT = new 
SimpleDateFormat("-MM-dd HH:mm:ss");
   private static final String NOW_TIME = FORMAT.format(DATE);
 
+  private static final String CREATE_QUERY = "CREATE TABLE if not exists 
mydb_sales_aggr_continuous_fact "
++ "(order_time timestamp, delivery_time timestamp, customer_id integer, 
product_id integer, promotion_id integer, "
++ "customer_city_id integer, production_city_id integer, delivery_city_id 
integer, unit_sales integer, "
++ "store_sales integer, store_cost integer, max_line_item_price integer, 
max_line_item_discount integer)";
   private static final String INSERT_QUERY = "INSERT INTO "
   + "mydb_sales_aggr_continuous_fact (order_time, delivery_time, 
customer_id, "
   + "product_id, promotion_id, customer_city_id, production_city_id, 
delivery_city_id, unit_sales, "
@@ -119,6 +123,7 @@ public class PopulateSampleMetastore {
 con.setAutoCommit(true);
 Statement statement = con.createStatement();
 try {
+  statement.execute(CREATE_QUERY);
   statement.execute(INSERT_QUERY);
 
 } finally {

http://git-wip-us.apache.org/repos/asf/lens/blob/2b86ac2b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
--
diff --git 
a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java 
b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
index e5a397b..1aa993b 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
@@ -79,6 +79,9 @@ public class SampleMetastore {
   public void createCubes() throws JAXBException, IOException {
 createCube("sample-cube.xml");
 createCube("sales-cube.xml");
+createCube("cube11.xml");
+createCube("cube22.xml");
+createCube("cube33.xml");
   }
 
   private void createDimension(String dimensionSpec) {

http://git-wip-us.apache.org/repos/asf/lens/blob/2b86ac2b/lens-examples/src/main/resources/sales-aggr-fact2-local4.data
--
diff --git a/lens-examples/src/main/resources/sales-aggr-fact2-local4.data 
b/lens-examples/src/main/resources/sales-aggr-fact2-local4.data
new file mode 100644
index 000..407b097
--- /dev/null
+++ b/lens-examples/src/main/resources/sales-aggr-fact2-local4.data
@@ -0,0 

lens git commit: LENS-1471: JoinCandidate's children should not share same storage candidate

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master a8d23df80 -> caf7a2c0b


LENS-1471: JoinCandidate's children should not share same storage candidate


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/caf7a2c0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/caf7a2c0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/caf7a2c0

Branch: refs/heads/master
Commit: caf7a2c0b045ce409d254c1ff26ffd79d01fee52
Parents: a8d23df
Author: Rajat Khandelwal 
Authored: Tue Sep 5 15:46:02 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Sep 5 15:46:02 2017 +0530

--
 .../lens/cube/parse/CandidateCoveringSetsResolver.java   | 8 ++--
 .../java/org/apache/lens/cube/parse/StorageCandidate.java| 2 +-
 .../main/java/org/apache/lens/cube/parse/UnionCandidate.java | 7 ++-
 3 files changed, 13 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/caf7a2c0/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1cff4a4..9125f4f 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -169,7 +169,8 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 }
   }
   @Deprecated
-  private List getCombinations(final List 
candidates, CubeQueryContext cubeql) {
+  private List getCombinations(final List 
candidates, CubeQueryContext cubeql)
+throws LensException {
 List combinations = new LinkedList<>();
 int size = candidates.size();
 int threshold = Double.valueOf(Math.pow(2, size)).intValue() - 1;
@@ -187,6 +188,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
   UnionCandidate uc = new UnionCandidate(individualCombinationList, 
cubeql);
   if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+uc.cloneChildren();
 combinations.add(uc);
   }
 }
@@ -207,7 +209,8 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
* @param cubeql
* @return
*/
-  private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
+  private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql)
+  throws LensException {
 LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
 List incompleteCombinations = Lists.newArrayList();
 incompleteCombinations.add(Lists.newArrayList());
@@ -221,6 +224,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 incompleteCombination.add(candidate);
 UnionCandidate unionCandidate = new 
UnionCandidate(incompleteCombination, cubeql);
 if (isCandidateCoveringTimeRanges(unionCandidate, 
cubeql.getTimeRanges())) {
+  unionCandidate.cloneChildren();
   unionCandidates.add(unionCandidate);
 } else {
   moreIncomplete.add(incompleteCombination);

http://git-wip-us.apache.org/repos/asf/lens/blob/caf7a2c0/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 27835b3..140b2c1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -169,7 +169,6 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
   rangeToPartitions.put(entry.getKey(), new 
LinkedHashSet<>(entry.getValue()));
 }
 this.rangeToExtraWhereFallBack = sc.rangeToExtraWhereFallBack;
-this.answerableMeasurePhraseIndices = sc.answerableMeasurePhraseIndices;
   }
 
   public StorageCandidate(CubeInterface cube, FactTable fact, String 
storageName, CubeQueryContext cubeQueryContext)
@@ -933,6 +932,7 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
 updatePeriodSpecificSc = copy();
 
updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(),
   storageName, period));
+updatePeriodSpecificSc.isStorageTblsAtUpdatePeriodLevel = false;
 updatePeriodSpecificSc.truncatePartitions(period);
 

lens git commit: LENS-1439: Having clause getting skipped if query goes to a segmentation containing single segment

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master b4cc2ed8a -> a8d23df80


LENS-1439: Having clause getting skipped if query goes to a segmentation 
containing single segment


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a8d23df8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a8d23df8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a8d23df8

Branch: refs/heads/master
Commit: a8d23df80dcfc22b3d67938f8f522bbd33072466
Parents: b4cc2ed
Author: Rajat Khandelwal 
Authored: Mon Sep 4 10:29:02 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Sep 4 10:29:02 2017 +0530

--
 .../lens/cube/parse/StorageCandidateHQLContext.java| 13 -
 .../lens/cube/parse/TestCubeSegmentationRewriter.java  |  6 +++---
 .../schema/cubes/derived/union_join_ctx_der1.xml   |  1 +
 3 files changed, 16 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a8d23df8/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index 21cdb61..432c0e4 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.parse;
 
 import static 
org.apache.lens.cube.metadata.MetastoreConstants.VIRTUAL_FACT_FILTER;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -50,7 +51,17 @@ public class StorageCandidateHQLContext extends 
DimHQLContext {
 this.storageCandidate = storageCandidate;
 this.rootCubeQueryContext = rootCubeQueryContext;
 getCubeQueryContext().addRangeClauses(this);
-if (!Objects.equals(getStorageCandidate(), 
rootCubeQueryContext.getPickedCandidate())) {
+boolean setNullHaving = true;
+if (Objects.equals(getStorageCandidate(), 
rootCubeQueryContext.getPickedCandidate())) {
+  setNullHaving = false;
+} else {
+  Collection children = 
rootCubeQueryContext.getPickedCandidate().getChildren();
+  // children should not be null. Not checking for null here.
+  if (children.size() == 1 && Objects.equals(getStorageCandidate(), 
children.iterator().next())) {
+setNullHaving = false;
+  }
+}
+if (setNullHaving) {
   getQueryAst().setHavingAST(null);
 }
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/a8d23df8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
index 7e1714b..8a76f4a 100644
--- 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
+++ 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java
@@ -316,11 +316,11 @@ public class TestCubeSegmentationRewriter extends 
TestQueryRewrite {
 
   @Test
   public void testSegmentationWithSingleSegment() throws LensException {
-String userQuery = "select segmsr1 from basecube where " + TWO_DAYS_RANGE;
+String userQuery = "select zipcode, segmsr1 from basecube where " + 
TWO_DAYS_RANGE + " having segmsr1 > 10";
 String actual = rewrite(userQuery, getConf());
 String expected = getExpectedQuery("basecube",
-  "select sum(basecube.segmsr1) FROM ", null,
-  null,
+  "select basecube.zipcode, sum(basecube.segmsr1) FROM ", null,
+  "group by basecube.zipcode having sum(basecube.segmsr1) > 10",
   getWhereForDailyAndHourly2days("basecube", "c1_b1fact1"));
 compareQueries(actual, expected);
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/a8d23df8/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
--
diff --git 
a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml 
b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
index c23a029..5a600ae 100644
--- a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
+++ b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml
@@ -45,5 +45,6 @@
 union_join_ctx_dup_cityname
 d_time
 union_join_ctx_cityid
+zipcode
   
 

[1/3] lens git commit: LENS-1464: One or two queued queries failing with ConcurrentModificationException on restart

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master b3352997c -> ef6e59c61


LENS-1464: One or two queued queries failing with 
ConcurrentModificationException on restart


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/806703ef
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/806703ef
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/806703ef

Branch: refs/heads/master
Commit: 806703ef2d8180a3baee21a6f27a3b7bdc1f2f79
Parents: b335299
Author: Rajat Khandelwal 
Authored: Wed Aug 30 15:59:23 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 30 15:59:23 2017 +0530

--
 lens-server/pom.xml |  4 +--
 .../server/query/QueryExecutionServiceImpl.java | 32 +---
 2 files changed, 16 insertions(+), 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/806703ef/lens-server/pom.xml
--
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 34a7819..92e218f 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -402,8 +402,8 @@
   
 
 
-  1.7
-  1.7
+  1.8
+  1.8
   enunciate.xml
 
   

http://git-wip-us.apache.org/repos/asf/lens/blob/806703ef/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
 
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index e70d290..4d141e5 100644
--- 
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ 
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -3476,27 +3476,23 @@ public class QueryExecutionServiceImpl extends 
BaseLensService implements QueryE
 
 // Add resources if either they haven't been marked as added on the 
session, or if Hive driver says they need
 // to be added to the corresponding hive driver
-if (!hiveDriver.areDBResourcesAddedForSession(sessionIdentifier, 
ctx.getDatabase())) {
-  Collection dbResources = 
session.getDBResources(ctx.getDatabase());
-
-  if (CollectionUtils.isNotEmpty(dbResources)) {
-log.info("Proceeding to add resources for DB {} for query {} 
resources: {}", session.getCurrentDatabase(),
-  ctx.getLogHandle(), dbResources);
-
-List failedDBResources = addResources(dbResources, 
sessionHandle, hiveDriver);
-Iterator itr = dbResources.iterator();
-while (itr.hasNext()) {
-  ResourceEntry res = itr.next();
-  if (!failedDBResources.contains(res)) {
-itr.remove();
-  }
+synchronized (session) {
+  if (!hiveDriver.areDBResourcesAddedForSession(sessionIdentifier, 
ctx.getDatabase())) {
+Collection dbResources = 
session.getDBResources(ctx.getDatabase());
+
+if (CollectionUtils.isNotEmpty(dbResources)) {
+  log.info("Proceeding to add resources for DB {} for query {} 
resources: {}", session.getCurrentDatabase(),
+ctx.getLogHandle(), dbResources);
+
+  List failedDBResources = addResources(dbResources, 
sessionHandle, hiveDriver);
+  dbResources.removeIf(res -> !failedDBResources.contains(res));
+} else {
+  log.info("No need to add DB resources for session: {} db= {}", 
sessionIdentifier,
+session.getCurrentDatabase());
 }
-  } else {
-log.info("No need to add DB resources for session: {} db= {}", 
sessionIdentifier, session.getCurrentDatabase());
+hiveDriver.setResourcesAddedForSession(sessionIdentifier, 
ctx.getDatabase());
   }
-  hiveDriver.setResourcesAddedForSession(sessionIdentifier, 
ctx.getDatabase());
 }
-
 // Get pending session resources which needed to be added for this database
 Collection pendingResources =
   session.getPendingSessionResourcesForDatabase(ctx.getDatabase());

[2/3] lens git commit: LENS-1465: Sync queries blocking threads

[prongs]

LENS-1465: Sync queries blocking threads


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f4fdab01
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f4fdab01
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f4fdab01

Branch: refs/heads/master
Commit: f4fdab01395357159b2c641c8e6aa88223551838
Parents: 806703e
Author: Rajat Khandelwal 
Authored: Wed Aug 30 16:10:13 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 30 16:10:13 2017 +0530

--
 .../org/apache/lens/server/api/query/QueryContext.java   | 11 +--
 1 file changed, 5 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/f4fdab01/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 466fb25..8176f03 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -198,7 +198,7 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   @Getter
   @Setter
   private transient Future queryLauncher;
-  transient List driverStatusUpdateListeners 
= Lists.newArrayList();
+  transient List driverStatusUpdateListeners 
= Lists.newCopyOnWriteArrayList();
   @Getter
   @Setter
   List failedAttempts = Lists.newArrayList();
@@ -572,10 +572,9 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   getDriverStatus().setStatusMessage("Query " + getQueryHandleString() + " 
" + state.name().toLowerCase());
 }
 getDriverStatus().setState(state);
-synchronized (this) {
-  for (QueryDriverStatusUpdateListener listener : 
this.driverStatusUpdateListeners) {
-listener.onDriverStatusUpdated(getQueryHandle(), getDriverStatus());
-  }
+
+for (QueryDriverStatusUpdateListener listener : 
this.driverStatusUpdateListeners) {
+  listener.onDriverStatusUpdated(getQueryHandle(), getDriverStatus());
 }
   }
 
@@ -588,7 +587,7 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   }
 
 
-  public synchronized void 
registerStatusUpdateListener(QueryDriverStatusUpdateListener 
driverStatusUpdateListener) {
+  public void registerStatusUpdateListener(QueryDriverStatusUpdateListener 
driverStatusUpdateListener) {
 this.driverStatusUpdateListeners.add(driverStatusUpdateListener);
   }
 

[3/3] lens git commit: LENS-1467: CubeQueryContext.getAllFilters is returning incorrect list of filters in case there is an "OR" in the filters

[prongs]

LENS-1467: CubeQueryContext.getAllFilters is returning incorrect list of 
filters in case there is an "OR" in the filters


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ef6e59c6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ef6e59c6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ef6e59c6

Branch: refs/heads/master
Commit: ef6e59c61d9b281332911a5d282723d29cd700c3
Parents: f4fdab0
Author: Rajat Khandelwal 
Authored: Wed Aug 30 16:10:54 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 30 16:10:55 2017 +0530

--
 .../lens/cube/parse/CubeQueryContext.java   | 67 ++-
 .../cube/parse/StorageCandidateHQLContext.java  |  3 +-
 .../apache/lens/cube/parse/join/JoinClause.java |  9 +++
 .../lens/cube/parse/CubeQueryContextTest.java   | 70 
 4 files changed, 100 insertions(+), 49 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/ef6e59c6/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 8b9583a..bff5c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -49,7 +49,6 @@ import org.apache.lens.cube.metadata.*;
 import org.apache.lens.cube.metadata.join.TableRelationship;
 import org.apache.lens.cube.parse.join.AutoJoinContext;
 import org.apache.lens.cube.parse.join.JoinClause;
-import org.apache.lens.cube.parse.join.JoinTree;
 import org.apache.lens.cube.parse.join.JoinUtils;
 import org.apache.lens.server.api.error.LensException;
 
@@ -1137,14 +1136,13 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 return ImmutableSet.copyOf(this.queriedTimeDimCols);
   }
 
-  String getWhere(StorageCandidateHQLContext sc, AutoJoinContext autoJoinCtx,
-ASTNode node, String cubeAlias,
-boolean shouldReplaceDimFilter, String storageTable,
-Map dimToQuery) throws LensException {
+  String getWhere(StorageCandidateHQLContext sc, AutoJoinContext autoJoinCtx, 
String cubeAlias,
+boolean shouldReplaceDimFilter, Map dimToQuery) 
throws LensException {
 String whereString;
 if (autoJoinCtx != null && shouldReplaceDimFilter) {
   List allfilters = new ArrayList<>();
-  getAllFilters(node, cubeAlias, allfilters, 
autoJoinCtx.getJoinClause(sc.getStorageCandidate()), dimToQuery);
+  getAllFilters(sc.getQueryAst().getWhereAST(), cubeAlias, allfilters,
+autoJoinCtx.getJoinClause(sc.getStorageCandidate()), dimToQuery);
   whereString = StringUtils.join(allfilters, " and ");
 } else {
   whereString = HQLParser.getString(sc.getQueryAst().getWhereAST());
@@ -1152,58 +1150,33 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 return whereString;
   }
 
-  private void getAllFilters(ASTNode node, String cubeAlias, List 
allFilters,
-JoinClause joinClause,  Map dimToQuery)
-throws LensException {
-
-if (node.getToken().getType() == HiveParser.KW_AND) {
-  // left child is and
-  if (node.getChild(0).getType() == HiveParser.KW_AND) {
-// take right corresponding to right
-String table = getTableFromFilterAST((ASTNode) node.getChild(1));
-allFilters.add(getFilter(table, cubeAlias, node, joinClause, 1, 
dimToQuery));
-  } else if (node.getChildCount() > 1) {
-for (int i = 0; i < node.getChildCount(); i++) {
-  String table = getTableFromFilterAST((ASTNode) node.getChild(i));
-  allFilters.add(getFilter(table, cubeAlias, node, joinClause, i, 
dimToQuery));
-}
+  protected static void getAllFilters(ASTNode node, String cubeAlias, 
List allFilters, JoinClause joinClause,
+Map dimToQuery) throws LensException {
+if (node.getToken().getType() == HiveParser.KW_AND || 
node.getToken().getType() == HiveParser.TOK_WHERE) {
+  for (int i = 0; i < node.getChildCount(); i++) {
+ASTNode child = (ASTNode) node.getChild(i);
+getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery);
   }
-} else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND
-  && node.getChild(0).getType() != HiveParser.KW_AND) {
-  // if node is the only child
-  allFilters.add(HQLParser.getString((ASTNode) node));
-}
-for (int i = 0; i < 

[4/4] lens git commit: LENS-1459: Time union candidate creation phase is creating a combination with redundant candidates

[prongs]

LENS-1459: Time union candidate creation phase is creating a combination with 
redundant candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/10eef27a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/10eef27a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/10eef27a

Branch: refs/heads/current-release-line
Commit: 10eef27a0dcbb1a3b918ec0ceeabefd206e13045
Parents: 6a45685
Author: Rajat Khandelwal 
Authored: Wed Aug 16 17:47:39 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Aug 21 13:01:04 2017 +0530

--
 .../org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/10eef27a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index de30703..1cff4a4 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -127,6 +127,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 // Get all covering fact sets
 //List unionCoveringSet = getCombinations(new 
ArrayList<>(allCandidatesPartiallyValid), cubeql);
 List unionCoveringSet = 
getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
+pruneRedundantUnionCoveringSets(unionCoveringSet);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // prune candidate set which doesn't contain any common measure i
@@ -154,7 +155,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-  @Deprecated
+
   private void pruneRedundantUnionCoveringSets(List 
candidates) {
 for (int i = 0; i < candidates.size(); i++) {
   UnionCandidate current = candidates.get(i);

[3/4] lens git commit: LENS-1456: Fix Deadlock in jdbcdriver

[prongs]

LENS-1456: Fix Deadlock in jdbcdriver


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6a45685b
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6a45685b
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6a45685b

Branch: refs/heads/current-release-line
Commit: 6a45685b6baec3db0aaac50065c7c1adf57fdf3f
Parents: cb92ec5
Author: Rajat Khandelwal 
Authored: Fri Aug 11 15:52:48 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Aug 21 13:01:00 2017 +0530

--
 .../java/org/apache/lens/server/api/query/QueryContext.java  | 8 +++-
 1 file changed, 3 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/6a45685b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index ccdef87..466fb25 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -572,7 +572,7 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   getDriverStatus().setStatusMessage("Query " + getQueryHandleString() + " 
" + state.name().toLowerCase());
 }
 getDriverStatus().setState(state);
-synchronized (this.driverStatusUpdateListeners) {
+synchronized (this) {
   for (QueryDriverStatusUpdateListener listener : 
this.driverStatusUpdateListeners) {
 listener.onDriverStatusUpdated(getQueryHandle(), getDriverStatus());
   }
@@ -588,10 +588,8 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   }
 
 
-  public void registerStatusUpdateListener(QueryDriverStatusUpdateListener 
driverStatusUpdateListener) {
-synchronized (this.driverStatusUpdateListeners) {
-  this.driverStatusUpdateListeners.add(driverStatusUpdateListener);
-}
+  public synchronized void 
registerStatusUpdateListener(QueryDriverStatusUpdateListener 
driverStatusUpdateListener) {
+this.driverStatusUpdateListeners.add(driverStatusUpdateListener);
   }
 
   @Override

[1/4] lens git commit: LENS-1450 : Fix duplicate filters in query writing

[prongs]

Repository: lens
Updated Branches:
  refs/heads/current-release-line 1f9f12f25 -> 10eef27a0


LENS-1450 : Fix duplicate filters in query writing


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/05c364e2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/05c364e2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/05c364e2

Branch: refs/heads/current-release-line
Commit: 05c364e24820b8b6e10a226a523429ca35f9e4d4
Parents: 1f9f12f
Author: Rajitha R 
Authored: Mon Aug 7 17:29:26 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Aug 21 13:00:48 2017 +0530

--
 .../java/org/apache/lens/cube/parse/CubeQueryContext.java |  3 ++-
 .../java/org/apache/lens/cube/parse/TestCubeRewriter.java | 10 +-
 2 files changed, 11 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/05c364e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b5330a7..8b9583a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,8 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 }
   }
 } else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND) {
+&& node.getToken().getType() != HiveParser.KW_AND
+  && node.getChild(0).getType() != HiveParser.KW_AND) {
   // if node is the only child
   allFilters.add(HQLParser.getString((ASTNode) node));
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/05c364e2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 1c75e6c..bc0fa2e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -509,8 +509,16 @@ public class TestCubeRewriter extends TestQueryRewrite {
 conf.setBoolean(DISABLE_AUTO_JOINS, false);
 conf.setBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, true);
 
+// No filter
+String hql = rewrite("select cityid , msr2 from testCube where " + 
TWO_DAYS_RANGE, conf);
+
+String expectedQuery =  getExpectedQuery(TEST_CUBE_NAME, "select 
(testcube.cityid) as `cityid`, "
++ "sum((testcube.msr2)) as `msr2` from ", null, "group by 
testcube.cityid",
+  getWhereForHourly2days(TEST_CUBE_NAME, "c3_testfact2_raw"));
+compareQueries(expectedQuery, hql);
+
 // filter with =
-String hql = rewrite(
+hql = rewrite(
 "select cubecountry.name, msr2 from" + " testCube" + " where 
cubecountry.region = 'asia' and "
 + TWO_DAYS_RANGE, conf);
 String filterSubquery = "testcube.countryid in ( select id from 
TestQueryRewrite.c3_countrytable_partitioned "

[2/4] lens git commit: LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for queued queries after server restart

[prongs]

LENS-1458 : RESULT_SET_PARENT_DIR set at driver level getting overwritten for 
queued queries after server restart


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cb92ec5c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cb92ec5c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cb92ec5c

Branch: refs/heads/current-release-line
Commit: cb92ec5cdafdee653465c7b6038850992123136b
Parents: 05c364e
Author: Sushil Mohanty 
Authored: Mon Aug 7 18:38:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Aug 21 13:00:55 2017 +0530

--
 .../apache/lens/server/api/query/DriverSelectorQueryContext.java   | 2 +-
 .../src/test/java/org/apache/lens/server/TestServerRestart.java| 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/cb92ec5c/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 41e533e..9cec42a 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -254,7 +254,7 @@ public class DriverSelectorQueryContext {
   }
 
   public void setDriverConf(LensDriver driver, Configuration conf) {
-driverQueryContextMap.get(driver).setDriverSpecificConf(conf);
+driverQueryContextMap.get(driver).setDriverSpecificConf(mergeConf(driver, 
conf));
   }
 
   public void setDriverCost(LensDriver driver, QueryCost cost) {

http://git-wip-us.apache.org/repos/asf/lens/blob/cb92ec5c/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java 
b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 20aa8cb..8b7728e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -39,6 +39,7 @@ import org.apache.lens.api.APIResult.Status;
 import org.apache.lens.api.query.*;
 import org.apache.lens.api.result.LensAPIResult;
 import org.apache.lens.driver.hive.TestRemoteHiveDriver;
+import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.QueryContext;
 import org.apache.lens.server.api.query.QueryExecutionService;
@@ -245,6 +246,7 @@ public class TestServerRestart extends 
LensAllApplicationJerseyTest {
 assertEquals(lensQueryConf.getProperty(KEY_POST_SELECT), 
VALUE_POST_SELECT);
 
 if (afterRestart) {
+  assertEquals(driverConf.get(LensConfConstants.RESULT_SET_PARENT_DIR), 
"target/hive-lens-results");
   //This will be unavailable since if was not updated in LensConf by 
MockDriverQueryHook
   assertNull(driverConf.get(UNSAVED_KEY_POST_SELECT));
 } else {

lens git commit: LENS-1459: Time union candidate creation phase is creating a combination with redundant candidates

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 66ff2fd50 -> e6f1ce0e2


LENS-1459: Time union candidate creation phase is creating a combination with 
redundant candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e6f1ce0e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e6f1ce0e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e6f1ce0e

Branch: refs/heads/master
Commit: e6f1ce0e2927d84a9cd2800f7630305114af1bb4
Parents: 66ff2fd
Author: Rajat Khandelwal 
Authored: Wed Aug 16 17:47:39 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 16 17:47:39 2017 +0530

--
 .../org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/e6f1ce0e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index de30703..1cff4a4 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -127,6 +127,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 // Get all covering fact sets
 //List unionCoveringSet = getCombinations(new 
ArrayList<>(allCandidatesPartiallyValid), cubeql);
 List unionCoveringSet = 
getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
+pruneRedundantUnionCoveringSets(unionCoveringSet);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // prune candidate set which doesn't contain any common measure i
@@ -154,7 +155,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-  @Deprecated
+
   private void pruneRedundantUnionCoveringSets(List 
candidates) {
 for (int i = 0; i < candidates.size(); i++) {
   UnionCandidate current = candidates.get(i);

lens git commit: Revert "LENS-1450: filters are getting replicated in jdbc driver query"

[prongs]

Repository: lens
Updated Branches:
  refs/heads/current-release-line f43c20512 -> 1f9f12f25


Revert "LENS-1450: filters are getting replicated in jdbc driver query"

This reverts commit 63e0b691741b9ac045cbfe6dd988000988077800.


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1f9f12f2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1f9f12f2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1f9f12f2

Branch: refs/heads/current-release-line
Commit: 1f9f12f25b26ff00443410c7318e785499cd19f6
Parents: f43c205
Author: Rajat Khandelwal 
Authored: Wed Aug 2 20:47:38 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 2 20:47:53 2017 +0530

--
 .../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/1f9f12f2/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index fb49b6d..b5330a7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,7 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 }
   }
 } else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND && 
node.getChildCount() == 0) {
+&& node.getToken().getType() != HiveParser.KW_AND) {
   // if node is the only child
   allFilters.add(HQLParser.getString((ASTNode) node));
 }

lens git commit: LENS-1457: ExpressionResolver pruning expression when some tables don't have ref columns of the expressions

[prongs]

Repository: lens
Updated Branches:
  refs/heads/current-release-line cdd7b0999 -> f43c20512


LENS-1457: ExpressionResolver pruning expression when some tables don't have 
ref columns of the expressions


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f43c2051
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f43c2051
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f43c2051

Branch: refs/heads/current-release-line
Commit: f43c20512facb9686316b5b4aed1af9515d33fb5
Parents: cdd7b09
Author: Rajat Khandelwal 
Authored: Mon Jul 31 15:16:35 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Aug 2 20:47:04 2017 +0530

--
 .../java/org/apache/lens/cube/parse/CandidateTable.java   | 10 ++
 .../org/apache/lens/cube/parse/ExpressionResolver.java|  4 
 2 files changed, 6 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/f43c2051/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
index c909545..40022f0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
@@ -22,11 +22,12 @@ import java.util.Collection;
 import java.util.Set;
 
 import org.apache.lens.cube.metadata.AbstractCubeTable;
+import org.apache.lens.cube.metadata.Named;
 
 /**
  * Candidate table interface
  */
-public interface CandidateTable {
+public interface CandidateTable extends Named {
 
   /**
* Get storage string of the base table alias passed
@@ -57,13 +58,6 @@ public interface CandidateTable {
   AbstractCubeTable getBaseTable();
 
   /**
-   * Get name of the candidate table
-   *
-   * @return name
-   */
-  String getName();
-
-  /**
* Get columns of candidate table
*
* @return set or list of columns

http://git-wip-us.apache.org/repos/asf/lens/blob/f43c2051/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 4680766..8906fae 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -19,6 +19,8 @@
 
 package org.apache.lens.cube.parse;
 
+import static java.util.stream.Collectors.toSet;
+
 import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
 import java.util.*;
@@ -557,6 +559,8 @@ class ExpressionResolver implements ContextRewriter {
 // Remove expressions for which denormalized columns are no more 
reachable
 esc.getDeNormCtx().pruneReferences(cubeql);
 if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
+  && esc.getDeNormCtx().getTableToRefCols().keySet().containsAll(
+
ec.getEvaluableExpressions().keySet().stream().map(Named::getName).collect(toSet()))
   && esc.getDeNormCtx().getTableToRefCols().keySet().stream()
   
.map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty))
 {
   log.info("Removing expression {} as all tables have non 
reachable fields", esc);

lens git commit: LENS-1457: ExpressionResolver pruning expression when some tables don't have ref columns of the expressions

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 95b91f90d -> 1bafd570f


LENS-1457: ExpressionResolver pruning expression when some tables don't have 
ref columns of the expressions


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/1bafd570
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/1bafd570
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/1bafd570

Branch: refs/heads/master
Commit: 1bafd570fb8ccfd85be782dd9f63a37e3f26ead4
Parents: 95b91f9
Author: Rajat Khandelwal 
Authored: Mon Jul 31 15:16:35 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Jul 31 15:16:35 2017 +0530

--
 .../java/org/apache/lens/cube/parse/CandidateTable.java   | 10 ++
 .../org/apache/lens/cube/parse/ExpressionResolver.java|  4 
 2 files changed, 6 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/1bafd570/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
index c909545..40022f0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTable.java
@@ -22,11 +22,12 @@ import java.util.Collection;
 import java.util.Set;
 
 import org.apache.lens.cube.metadata.AbstractCubeTable;
+import org.apache.lens.cube.metadata.Named;
 
 /**
  * Candidate table interface
  */
-public interface CandidateTable {
+public interface CandidateTable extends Named {
 
   /**
* Get storage string of the base table alias passed
@@ -57,13 +58,6 @@ public interface CandidateTable {
   AbstractCubeTable getBaseTable();
 
   /**
-   * Get name of the candidate table
-   *
-   * @return name
-   */
-  String getName();
-
-  /**
* Get columns of candidate table
*
* @return set or list of columns

http://git-wip-us.apache.org/repos/asf/lens/blob/1bafd570/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 4680766..8906fae 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -19,6 +19,8 @@
 
 package org.apache.lens.cube.parse;
 
+import static java.util.stream.Collectors.toSet;
+
 import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
 import java.util.*;
@@ -557,6 +559,8 @@ class ExpressionResolver implements ContextRewriter {
 // Remove expressions for which denormalized columns are no more 
reachable
 esc.getDeNormCtx().pruneReferences(cubeql);
 if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
+  && esc.getDeNormCtx().getTableToRefCols().keySet().containsAll(
+
ec.getEvaluableExpressions().keySet().stream().map(Named::getName).collect(toSet()))
   && esc.getDeNormCtx().getTableToRefCols().keySet().stream()
   
.map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty))
 {
   log.info("Removing expression {} as all tables have non 
reachable fields", esc);

[2/3] lens git commit: LENS-1454: Time Covering set algorithm is skipping some combinations

[prongs]

LENS-1454: Time Covering set algorithm is skipping some combinations


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8b412023
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8b412023
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8b412023

Branch: refs/heads/current-release-line
Commit: 8b4120237af9aaca3a6934650ccfeac24bc62c59
Parents: 65ca04b
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:10:52 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:13:07 2017 +0530

--
 .../lens/cube/parse/CandidateCoveringSetsResolver.java  | 12 +++-
 1 file changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/8b412023/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 69d9562..4066cf7 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -209,21 +209,23 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
 LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
 List incompleteCombinations = Lists.newArrayList();
+incompleteCombinations.add(Lists.newArrayList());
 List unionCandidates = Lists.newArrayList();
 
 while(!candidateLinkedList.isEmpty()) {
+  List moreIncomplete = Lists.newArrayList();
   Candidate candidate = candidateLinkedList.remove();
-  incompleteCombinations.add(Lists.newArrayList());
-  Iterator iter = incompleteCombinations.iterator();
-  while(iter.hasNext()) {
-List incompleteCombination = iter.next();
+  for (List combination : incompleteCombinations) {
+List incompleteCombination = 
Lists.newArrayList(combination);
 incompleteCombination.add(candidate);
 UnionCandidate unionCandidate = new 
UnionCandidate(incompleteCombination, cubeql);
 if (isCandidateCoveringTimeRanges(unionCandidate, 
cubeql.getTimeRanges())) {
   unionCandidates.add(unionCandidate);
-  iter.remove();
+} else {
+  moreIncomplete.add(incompleteCombination);
 }
   }
+  incompleteCombinations.addAll(moreIncomplete);
 }
 return unionCandidates;
   }

[3/3] lens git commit: LENS-1453: Method metrics names duplicate for segmentation inner rewrite

[prongs]

LENS-1453: Method metrics names duplicate for segmentation inner rewrite


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/65ca04b5
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/65ca04b5
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/65ca04b5

Branch: refs/heads/current-release-line
Commit: 65ca04b598bd29201a6670b7a51679f9ccb9ccc6
Parents: 6dca446
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:09:58 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:13:07 2017 +0530

--
 .../org/apache/lens/cube/parse/SegmentationCandidate.java   | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/65ca04b5/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index a2bd485..35638f3 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -52,6 +52,7 @@ import org.apache.lens.cube.metadata.MetastoreUtil;
 import org.apache.lens.cube.metadata.Segment;
 import org.apache.lens.cube.metadata.Segmentation;
 import org.apache.lens.cube.metadata.TimeRange;
+import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.commons.lang3.tuple.Pair;
@@ -141,7 +142,13 @@ public class SegmentationCandidate implements Candidate {
 });
   addCubeNameAndAlias(ast, innerCube);
   trimHavingAndOrderby(ast, innerCube);
-  CubeQueryRewriter rewriter = new CubeQueryRewriter(conf, hconf);
+  Configuration innerConf = conf;
+  if (conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) != null) 
{
+innerConf = new Configuration(conf);
+innerConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY,
+  conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) + "-" + 
segment.getName());
+  }
+  CubeQueryRewriter rewriter = new CubeQueryRewriter(innerConf, hconf);
   CubeQueryContext ctx = rewriter.rewrite(ast);
   cubeQueryContextMap.put(segment.getName(), ctx);
   if (!ctx.getCandidates().isEmpty()) {

[1/3] lens git commit: LENS-1455: Getting expression ast to generate default expression should consider all expression contexts

[prongs]

Repository: lens
Updated Branches:
  refs/heads/current-release-line 6dca44661 -> cdd7b0999


LENS-1455: Getting expression ast to generate default expression should 
consider all expression contexts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cdd7b099
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cdd7b099
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cdd7b099

Branch: refs/heads/current-release-line
Commit: cdd7b09995b765783867812e5b981c27f4938371
Parents: 8b41202
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:11:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:13:07 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java  |  3 +++
 .../apache/lens/cube/parse/ExpressionResolver.java | 17 -
 .../lens/cube/parse/SegmentationCandidate.java |  5 +
 3 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 4066cf7..de30703 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -255,6 +255,9 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 i.remove();
   }
 }
+// sorting will make sure storage candidates come before complex 
candidates.
+// ensuring maximum columns get selected from simpler candidates.
+ucSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // Sets that contain all measures or no measures are removed from 
iteration.
 // find other facts
 for (Iterator i = ucSet.iterator(); i.hasNext();) {

http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 66b043e..4680766 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
 
 import org.antlr.runtime.CommonToken;
 
+import com.google.common.collect.Sets;
 import lombok.*;
 import lombok.extern.slf4j.Slf4j;
 
@@ -501,15 +502,21 @@ class ExpressionResolver implements ContextRewriter {
   }
 }
 
-private ASTNode getExprAst(ExpressionContext ec) {
+private ASTNode getExprAst(ExpressionContext expressionContext) {
   Set scSet = 
CandidateUtil.getStorageCandidates(cubeql.getCandidates());
-  Set storageTableNames = new HashSet();
+  Set storageTableNames = new HashSet<>();
+  Set expressionContexts = Sets.newLinkedHashSet();
+  expressionContexts.add(expressionContext);
   for (StorageCandidate sc : scSet) {
 storageTableNames.add(sc.getStorageTable());
+expressionContexts.add(sc.getCubeQueryContext().getExprCtx()
+  .getExpressionContext(expressionContext.getExprCol().getName(), 
expressionContext.getSrcAlias()));
   }
-  for (CandidateTable table : ec.evaluableExpressions.keySet()) {
-if (storageTableNames.contains(table.getStorageTable())) {
-  return  
MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+  for (ExpressionContext ec : expressionContexts) {
+for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+  if (storageTableNames.contains(table.getStorageTable())) {
+return 
MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+  }
 }
   }
   return null;

http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index 35638f3..055d6ef 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ 

[1/3] lens git commit: LENS-1453: Method metrics names duplicate for segmentation inner rewrite

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master d4236668c -> 95b91f90d


LENS-1453: Method metrics names duplicate for segmentation inner rewrite


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0b265c81
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0b265c81
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0b265c81

Branch: refs/heads/master
Commit: 0b265c81b312f410d4b4c6cde7f3a4596fb3fb79
Parents: d423666
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:09:58 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:09:58 2017 +0530

--
 .../org/apache/lens/cube/parse/SegmentationCandidate.java   | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/0b265c81/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index a2bd485..35638f3 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -52,6 +52,7 @@ import org.apache.lens.cube.metadata.MetastoreUtil;
 import org.apache.lens.cube.metadata.Segment;
 import org.apache.lens.cube.metadata.Segmentation;
 import org.apache.lens.cube.metadata.TimeRange;
+import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.commons.lang3.tuple.Pair;
@@ -141,7 +142,13 @@ public class SegmentationCandidate implements Candidate {
 });
   addCubeNameAndAlias(ast, innerCube);
   trimHavingAndOrderby(ast, innerCube);
-  CubeQueryRewriter rewriter = new CubeQueryRewriter(conf, hconf);
+  Configuration innerConf = conf;
+  if (conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) != null) 
{
+innerConf = new Configuration(conf);
+innerConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY,
+  conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) + "-" + 
segment.getName());
+  }
+  CubeQueryRewriter rewriter = new CubeQueryRewriter(innerConf, hconf);
   CubeQueryContext ctx = rewriter.rewrite(ast);
   cubeQueryContextMap.put(segment.getName(), ctx);
   if (!ctx.getCandidates().isEmpty()) {

[3/3] lens git commit: LENS-1455: Getting expression ast to generate default expression should consider all expression contexts

[prongs]

LENS-1455: Getting expression ast to generate default expression should 
consider all expression contexts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/95b91f90
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/95b91f90
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/95b91f90

Branch: refs/heads/master
Commit: 95b91f90d41a8dba2aa5d85ff170ba8d36f56b21
Parents: e977ce6
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:11:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:11:15 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java  |  3 +++
 .../apache/lens/cube/parse/ExpressionResolver.java | 17 -
 .../lens/cube/parse/SegmentationCandidate.java |  5 +
 3 files changed, 20 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/95b91f90/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 4066cf7..de30703 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -255,6 +255,9 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 i.remove();
   }
 }
+// sorting will make sure storage candidates come before complex 
candidates.
+// ensuring maximum columns get selected from simpler candidates.
+ucSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // Sets that contain all measures or no measures are removed from 
iteration.
 // find other facts
 for (Iterator i = ucSet.iterator(); i.hasNext();) {

http://git-wip-us.apache.org/repos/asf/lens/blob/95b91f90/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 66b043e..4680766 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
 
 import org.antlr.runtime.CommonToken;
 
+import com.google.common.collect.Sets;
 import lombok.*;
 import lombok.extern.slf4j.Slf4j;
 
@@ -501,15 +502,21 @@ class ExpressionResolver implements ContextRewriter {
   }
 }
 
-private ASTNode getExprAst(ExpressionContext ec) {
+private ASTNode getExprAst(ExpressionContext expressionContext) {
   Set scSet = 
CandidateUtil.getStorageCandidates(cubeql.getCandidates());
-  Set storageTableNames = new HashSet();
+  Set storageTableNames = new HashSet<>();
+  Set expressionContexts = Sets.newLinkedHashSet();
+  expressionContexts.add(expressionContext);
   for (StorageCandidate sc : scSet) {
 storageTableNames.add(sc.getStorageTable());
+expressionContexts.add(sc.getCubeQueryContext().getExprCtx()
+  .getExpressionContext(expressionContext.getExprCol().getName(), 
expressionContext.getSrcAlias()));
   }
-  for (CandidateTable table : ec.evaluableExpressions.keySet()) {
-if (storageTableNames.contains(table.getStorageTable())) {
-  return  
MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+  for (ExpressionContext ec : expressionContexts) {
+for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+  if (storageTableNames.contains(table.getStorageTable())) {
+return 
MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+  }
 }
   }
   return null;

http://git-wip-us.apache.org/repos/asf/lens/blob/95b91f90/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index 35638f3..055d6ef 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -273,6 +273,11 @@ public class SegmentationCandidate implements Candidate {
   }

[2/3] lens git commit: LENS-1454: Time Covering set algorithm is skipping some combinations

[prongs]

LENS-1454: Time Covering set algorithm is skipping some combinations


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e977ce6e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e977ce6e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e977ce6e

Branch: refs/heads/master
Commit: e977ce6e5a97015fedf24520d0632cd981a0cc5f
Parents: 0b265c8
Author: Rajat Khandelwal 
Authored: Thu Jul 20 16:10:52 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 20 16:10:52 2017 +0530

--
 .../lens/cube/parse/CandidateCoveringSetsResolver.java  | 12 +++-
 1 file changed, 7 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/e977ce6e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 69d9562..4066cf7 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -209,21 +209,23 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
 LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
 List incompleteCombinations = Lists.newArrayList();
+incompleteCombinations.add(Lists.newArrayList());
 List unionCandidates = Lists.newArrayList();
 
 while(!candidateLinkedList.isEmpty()) {
+  List moreIncomplete = Lists.newArrayList();
   Candidate candidate = candidateLinkedList.remove();
-  incompleteCombinations.add(Lists.newArrayList());
-  Iterator iter = incompleteCombinations.iterator();
-  while(iter.hasNext()) {
-List incompleteCombination = iter.next();
+  for (List combination : incompleteCombinations) {
+List incompleteCombination = 
Lists.newArrayList(combination);
 incompleteCombination.add(candidate);
 UnionCandidate unionCandidate = new 
UnionCandidate(incompleteCombination, cubeql);
 if (isCandidateCoveringTimeRanges(unionCandidate, 
cubeql.getTimeRanges())) {
   unionCandidates.add(unionCandidate);
-  iter.remove();
+} else {
+  moreIncomplete.add(incompleteCombination);
 }
   }
+  incompleteCombinations.addAll(moreIncomplete);
 }
 return unionCandidates;
   }

[04/12] lens git commit: LENS-1444: Optimize the algorithm of finding all eligible union candidates

[prongs]

LENS-1444: Optimize the algorithm of finding all eligible union candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a7f407bc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a7f407bc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a7f407bc

Branch: refs/heads/current-release-line
Commit: a7f407bcb59ffa84c8ab6e830ba98aee81516085
Parents: c174583
Author: Rajat Khandelwal 
Authored: Fri Jun 23 16:39:37 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:52 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java  | 17 -
 1 file changed, 4 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a7f407bc/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 61c28c6..8e07162 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -106,16 +106,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return true;
   }
 
-  private void pruneUnionCandidatesNotCoveringAllRanges(List 
ucs, CubeQueryContext cubeql) {
-for (Iterator itr = ucs.iterator(); itr.hasNext();) {
-  UnionCandidate uc = itr.next();
-  if (!isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
-itr.remove();
-cubeql.addCandidatePruningMsg(uc, 
CandidateTablePruneCause.storageNotAvailableInRange(cubeql.getTimeRanges()));
-  }
-}
-  }
-
   private List resolveTimeRangeCoveringFactSet(CubeQueryContext 
cubeql,
   Set queriedMsrs, List 
qpcList) throws LensException {
 List candidateSet = new ArrayList<>();
@@ -138,8 +128,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
-// prune non covering sets
-pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql);
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
@@ -197,7 +185,10 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 clonedI = clonedI >>> 1;
 --count;
   }
-  combinations.add(new UnionCandidate(individualCombinationList, cubeql));
+  UnionCandidate uc = new UnionCandidate(individualCombinationList, 
cubeql);
+  if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+combinations.add(uc);
+  }
 }
 return combinations;
   }

[02/12] lens git commit: LENS-1442: Optimize algorithm of CandidateCoveringSetResolver

[prongs]

LENS-1442: Optimize algorithm of CandidateCoveringSetResolver


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/9da5b40e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/9da5b40e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/9da5b40e

Branch: refs/heads/current-release-line
Commit: 9da5b40e3afbe966256a601bae6204afd5d3e992
Parents: c2a9c93
Author: Rajat Khandelwal 
Authored: Tue Jun 20 16:09:32 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:49 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java| 15 +--
 1 file changed, 9 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/9da5b40e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1e9873f..61c28c6 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -208,21 +208,24 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 List ucSet = new ArrayList<>(candidates);
 // Check if a single set can answer all the measures and exprsWithMeasures
 for (Iterator i = ucSet.iterator(); i.hasNext();) {
-  boolean evaluable = false;
+  boolean allEvaluable = true;
+  boolean anyEvaluable = false;
   Candidate uc = i.next();
   for (QueriedPhraseContext msr : msrs) {
-evaluable = uc.isPhraseAnswerable(msr);
-if (!evaluable) {
-  break;
-}
+boolean evaluable = uc.isPhraseAnswerable(msr);
+allEvaluable &= evaluable;
+anyEvaluable |= evaluable;
   }
-  if (evaluable) {
+  if (allEvaluable) {
 // single set can answer all the measures as an UnionCandidate
 List one = new ArrayList<>();
 one.add(uc);
 msrCoveringSets.add(one);
 i.remove();
   }
+  if (!anyEvaluable) { // none evaluable
+i.remove();
+  }
 }
 // Sets that contain all measures or no measures are removed from 
iteration.
 // find other facts

[09/12] lens git commit: LENS-1450: filters are getting replicated in jdbc driver query

[prongs]

LENS-1450: filters are getting replicated in jdbc driver query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/503a46dc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/503a46dc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/503a46dc

Branch: refs/heads/current-release-line
Commit: 503a46dc6c89598cc2ea557f2311e4bd26c3d9a1
Parents: dd3b1bd
Author: Rajitha R 
Authored: Wed Jul 5 16:49:07 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:58 2017 +0530

--
 .../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/503a46dc/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b5330a7..fb49b6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,7 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 }
   }
 } else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND) {
+&& node.getToken().getType() != HiveParser.KW_AND && 
node.getChildCount() == 0) {
   // if node is the only child
   allFilters.add(HQLParser.getString((ASTNode) node));
 }

[10/12] lens git commit: LENS-1449 : lens.query.result.parent.dir to be set at driver level.

[prongs]

LENS-1449 : lens.query.result.parent.dir to be set at driver level.


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/41051ea4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/41051ea4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/41051ea4

Branch: refs/heads/current-release-line
Commit: 41051ea47f164f004f5fccaa7ea91765007811fc
Parents: 503a46d
Author: Raghavendra Singh 
Authored: Thu Jul 6 14:03:59 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:59 2017 +0530

--
 .../api/query/DriverSelectorQueryContext.java   |  4 +++-
 .../lens/server/api/query/QueryContext.java |  5 +
 .../lens/server/query/TestQueryService.java |  6 --
 .../lens/server/query/TestResultFormatting.java | 20 +++-
 .../drivers/hive/hive1/hivedriver-site.xml  |  5 +
 .../drivers/hive/hive2/hivedriver-site.xml  |  6 ++
 .../drivers/jdbc/jdbc1/jdbcdriver-site.xml  |  5 +
 lens-server/src/test/resources/lens-site.xml|  5 -
 8 files changed, 47 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 8e431d1..41e533e 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -155,7 +155,9 @@ public class DriverSelectorQueryContext {
   private Configuration mergeConf(LensDriver driver, Configuration queryConf) {
 Configuration conf = new Configuration(driver.getConf());
 for (Map.Entry entry : queryConf) {
-  conf.set(entry.getKey(), entry.getValue());
+  if (!conf.getFinalParameters().contains(entry.getKey())) {
+conf.set(entry.getKey(), entry.getValue());
+  }
 }
 conf.setClassLoader(queryConf.getClassLoader());
 return conf;

http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 9923589..ccdef87 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -328,6 +328,11 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   }
 
   public String getResultSetParentDir() {
+if (getSelectedDriver() != null && 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) != null) {
+  log.info("Fetching Parent Dir from driver conf:- "
+  + 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR));
+  return 
getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR);
+}
 return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, 
LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT);
   }
 

http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java 
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 1149696..b7cdb88 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -848,9 +848,11 @@ public class TestQueryService extends LensJerseyTest {
 LensResultSet rs = queryService.getResultset(handle3);
 //check persisted result path
 String expectedPath =
-ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + 
handle3.getHandleIdString()
+
ctx3.getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" 
+ handle3.getHandleIdString()
 + ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN);
-assertTrue(((PersistentResultSet) 
rs).getOutputPath().endsWith(expectedPath));
+assertTrue(((PersistentResultSet) 
rs).getOutputPath().endsWith(expectedPath)
+, "Result 

[05/12] lens git commit: LENS-1445: Expression having reference column ends up rewriting wrong query

[prongs]

LENS-1445: Expression having reference column ends up rewriting wrong query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d49f45a0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d49f45a0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d49f45a0

Branch: refs/heads/current-release-line
Commit: d49f45a0f8c6665784a3770a534d6495c21fd1bc
Parents: a7f407b
Author: Sushil Mohanty 
Authored: Fri Jun 23 16:40:25 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:53 2017 +0530

--
 .../lens/cube/parse/UnionQueryWriter.java   | 79 +---
 .../parse/TestCubeSegmentationRewriter.java |  3 +-
 .../cube/parse/TestUnionAndJoinCandidates.java  | 26 +++
 .../resources/schema/cubes/base/basecube.xml| 11 +++
 .../resources/schema/cubes/base/testcube.xml|  1 +
 5 files changed, 92 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d49f45a0/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index cc0a2e5..f6c9ce1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -28,7 +28,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
 import java.util.*;
 
-import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -117,7 +118,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
* @return ASTNode
* @throws LensException
*/
-  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidate sc)
+  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
 if (cubeql.getHavingAST() != null) {
   ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
@@ -251,25 +252,15 @@ public class UnionQueryWriter extends SimpleHQLContext {
 for (StorageCandidateHQLContext sc : storageCandidates) {
   node = (ASTNode) 
sc.getQueryAst().getSelectAST().getChild(position).getChild(0);
   if (HQLParser.isAggregateAST(node) || HQLParser.hasAggregate(node)) {
-return MetastoreUtil.copyAST(node);
+if (!node.getChild(1).toString().equals(DEFAULT_MEASURE))  {
+  return MetastoreUtil.copyAST(node);
+}
   }
 }
 return MetastoreUtil.copyAST(node);
   }
 
   /**
-   *  Check if ASTNode is answerable by StorageCandidate
-   * @param sc
-   * @param node
-   * @return
-   */
-  private boolean isNodeNotAnswerableForStorageCandidate(StorageCandidate sc, 
ASTNode node) {
-Set cols = new LinkedHashSet<>();
-getAllColumnsOfNode(node, cols);
-return !sc.getColumns().containsAll(cols);
-  }
-
-  /**
* Set the default value "0.0" in the non answerable aggreagte expressions.
* @param node
* @param sc
@@ -467,7 +458,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
 // Iterate over the StorageCandidates and add non projected having columns 
in inner select ASTs
 for (StorageCandidateHQLContext sc : storageCandidates) {
   aliasDecider.setCounter(selectAliasCounter);
-  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, 
sc.getStorageCandidate());
+  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc);
 }
 removeRedundantProjectedPhrases();
   }
@@ -493,7 +484,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
   ASTNode child = (ASTNode) selectAST.getChild(i);
   ASTNode outerSelect = new ASTNode(child);
   ASTNode selectExprAST = (ASTNode) child.getChild(0);
-  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc.getStorageCandidate(), true,
+  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc, true,
   cubeql.getBaseCube().getDimAttributeNames());
   outerSelect.addChild(outerAST);
   // has an alias? add it
@@ -529,13 +520,14 @@ public class UnionQueryWriter extends SimpleHQLContext {
5. If given ast is memorized as mentioned in the above cases, return the 
mapping.
  */
   private ASTNode getOuterAST(ASTNode astNode, ASTNode innerSelectAST,
-  AliasDecider aliasDecider, StorageCandidate sc, boolean isSelectAst, 
Set dimensionSet)
+  

[12/12] lens git commit: LENS-1452: Optimize Time Union candidate Algorithm

[prongs]

LENS-1452: Optimize Time Union candidate Algorithm


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6dca4466
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6dca4466
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6dca4466

Branch: refs/heads/current-release-line
Commit: 6dca44661bf604ca1436c6cd1d3998405d0333a4
Parents: 3769ef0
Author: Rajat Khandelwal 
Authored: Mon Jul 10 16:52:54 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:43:03 2017 +0530

--
 .../parse/CandidateCoveringSetsResolver.java| 47 +---
 1 file changed, 41 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/6dca4466/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 8e07162..69d9562 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -26,6 +26,7 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.TimeRange;
 import org.apache.lens.server.api.error.LensException;
 
+import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -124,16 +125,14 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
 // Get all covering fact sets
-List unionCoveringSet =
-getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
+//List unionCoveringSet = getCombinations(new 
ArrayList<>(allCandidatesPartiallyValid), cubeql);
+List unionCoveringSet = 
getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
 }
-// prune redundant covering sets
-pruneRedundantUnionCoveringSets(unionCoveringSet);
 // pruing done in the previous steps, now create union candidates
 candidateSet.addAll(unionCoveringSet);
 updateQueriableMeasures(candidateSet, qpcList);
@@ -155,7 +154,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private void pruneRedundantUnionCoveringSets(List 
candidates) {
 for (int i = 0; i < candidates.size(); i++) {
   UnionCandidate current = candidates.get(i);
@@ -168,7 +167,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private List getCombinations(final List 
candidates, CubeQueryContext cubeql) {
 List combinations = new LinkedList<>();
 int size = candidates.size();
@@ -193,6 +192,42 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return combinations;
   }
 
+  /**
+   * The following function is iterative rewrite of the following 
tail-recursive implementation:
+   * (ignoring cubeql for clarity)
+   * getCombinations(candidates) = getCombinationsTailRecursive(emptyList(), 
candidates)
+   *
+   * getCombinationsTailRecursive(incompleteCombinations: 
List[List[Candidate]], candidates: List[Candidate]) =
+   *   head, tail = head and tail of linked List candidates
+   *   add head to all elements of incompleteCombinations.
+   *   complete = remove now complete combinations from incompleteCombinations
+   *   return complete ++ getCombinationsTailRecursive(incompleteCombinations, 
tail)
+   * @param candidates
+   * @param cubeql
+   * @return
+   */
+  private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
+LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
+List incompleteCombinations = Lists.newArrayList();
+List unionCandidates = Lists.newArrayList();
+
+while(!candidateLinkedList.isEmpty()) {
+  Candidate candidate = candidateLinkedList.remove();
+  incompleteCombinations.add(Lists.newArrayList());
+  Iterator iter = incompleteCombinations.iterator();
+  while(iter.hasNext()) {
+List incompleteCombination = iter.next();
+incompleteCombination.add(candidate);
+UnionCandidate unionCandidate = new 
UnionCandidate(incompleteCombination, cubeql);
+if 

[03/12] lens git commit: LENS-1443: Fallback ranges not working for virtual facts

[prongs]

LENS-1443: Fallback ranges not working for virtual facts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c174583f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c174583f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c174583f

Branch: refs/heads/current-release-line
Commit: c174583ff7946d0347e0a0a87272f42c0023aecf
Parents: 9da5b40
Author: Rajat Khandelwal 
Authored: Wed Jun 21 13:20:09 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:50 2017 +0530

--
 .../apache/lens/cube/metadata/FactTable.java| 38 ++--
 .../apache/lens/cube/parse/JoinCandidate.java   |  7 ++--
 .../lens/cube/parse/LeastPartitionResolver.java | 21 +--
 .../lens/cube/parse/StorageCandidate.java   |  2 +-
 .../lens/cube/parse/StorageTableResolver.java   |  5 +--
 5 files changed, 37 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index f87cf44..a463c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -35,132 +35,132 @@ public interface FactTable extends Named {
*
* @return Map of storage to set of update periods
*/
-  public Map getUpdatePeriods();
+  Map getUpdatePeriods();
 
   /**
* Cube to which this fact belongs to
*
* @return the cube string
*/
-  public String getCubeName();
+  String getCubeName();
 
   /**
* The set of Storage names
*
* @return set of strings
*/
-  public Set getStorages();
+  Set getStorages();
 
   /**
*The type of the fact
*
* @return table type {@link CubeTableType}
*/
-  public CubeTableType getTableType();
+  CubeTableType getTableType();
 
   /**
* Config properties
*
* @return map of string, string
*/
-  public Map getProperties();
+  Map getProperties();
 
   /**
* Valid columns of the fact
*
* @return list of column names
*/
-  public Set getValidColumns();
+  Set getValidColumns();
 
   /**
* Weight of the fact
*
* @return weight of the fact in double
*/
-  public double weight();
+  double weight();
 
   /**
* Set of all the columns names of the fact
*
* @return set of column names
*/
-  public Set getAllFieldNames();
+  Set getAllFieldNames();
 
   /**
*tag for checking data completeness
*
* @return Tag String
*/
-  public String getDataCompletenessTag();
+  String getDataCompletenessTag();
 
   /**
* List of columns of the fact
*
* @return set of {@link FieldSchema}
*/
-  public List getColumns();
+  List getColumns();
 
   /**
* Is Aggregated Fact
*
* @return true if fact is Aggregated , false otherwise
*/
-  public boolean isAggregated();
+  boolean isAggregated();
 
   /**
* Absolute start time of the fact
*
* @return Absolute Start time of the fact {@link Date}
*/
-  public Date getAbsoluteStartTime();
+  Date getAbsoluteStartTime();
 
   /**
* Relative start time of the fact
*
* @return Relative Start time of the fact {@link Date}
*/
-  public Date getRelativeStartTime();
+  Date getRelativeStartTime();
 
   /**
* Start time of the fact
*
* @return Start time of the fact {@link Date}
*/
-  public Date getStartTime();
+  Date getStartTime();
 
   /**
* Absolute end time of the fact
*
* @return Absolute End time of the fact {@link Date}
*/
-  public Date getAbsoluteEndTime();
+  Date getAbsoluteEndTime();
 
   /**
* Relative End time of the Fact
*
* @return Relative end time of the fact {@link Date}
*/
-  public Date getRelativeEndTime();
+  Date getRelativeEndTime();
 
   /**
* End time of the fact
*
* @return End time of the fact {@link Date}
*/
-  public Date getEndTime();
+  Date getEndTime();
 
   /**
* Is Virtual Fact
*
* @return true if fact is a virtual fact, false otherwise
*/
-  public boolean isVirtualFact();
+  boolean isVirtualFact();
 
   /**
* Storage name of the fact
*
* @return Storage name of the fact
*/
-  public String getSourceFactName();
+  String getSourceFactName();
 
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
--

[06/12] lens git commit: LENS-1437 : Missing unit test cases for virtual fact

[prongs]

LENS-1437 : Missing unit test cases for virtual fact


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/fbad3507
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/fbad3507
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/fbad3507

Branch: refs/heads/current-release-line
Commit: fbad350798a342a4de5e5b8b566a6f055338fde3
Parents: d49f45a
Author: Rajitha R 
Authored: Tue Jun 27 16:12:56 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:54 2017 +0530

--
 .../apache/lens/cube/parse/CubeTestSetup.java   |   4 +-
 .../lens/cube/parse/TestCubeRewriter.java   |  18 +--
 .../lens/cube/parse/TestVirtualFactQueries.java | 123 +++
 .../resources/schema/cubes/base/testcube2.xml   |  31 +
 .../resources/schema/cubes/base/virtualcube.xml |  23 +++-
 .../resources/schema/facts/testfact7_base.xml   |  67 ++
 .../resources/schema/facts/testfact8_base.xml   |  58 +
 .../resources/schema/facts/testfact9_base.xml   |  57 +
 .../schema/facts/virtual/virtualfact.xml|   2 +-
 9 files changed, 361 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 48869c2..0366e56 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -364,7 +364,7 @@ public class CubeTestSetup {
   // storageName[0] is hourly
   // storageName[1] is daily
   // storageName[2] is monthly
-  public static Map 
getWhereForMonthlyDailyAndHourly2months(String... storageTables) {
+  public static Map 
getWhereForMonthlyDailyAndHourly2months(String cubeName, String... 
storageTables) {
 Map storageTableToWhereClause = new LinkedHashMap();
 List hourlyparts = new ArrayList();
 List dailyparts = new ArrayList();
@@ -411,7 +411,7 @@ public class CubeTestSetup {
   tables.append(storageTables[0]);
 }
 Collections.sort(parts);
-storageTableToWhereClause.put(tables.toString(), 
StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
+storageTableToWhereClause.put(tables.toString(), 
StorageUtil.getWherePartClause("dt", cubeName, parts));
 return storageTableToWhereClause;
   }
 

http://git-wip-us.apache.org/repos/asf/lens/blob/fbad3507/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index d8c7335..1c75e6c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -145,22 +145,6 @@ public class TestCubeRewriter extends TestQueryRewrite {
 //assertNotNull(rewrittenQuery.getNonExistingParts());
   }
 
-
-  @Test
-  public void testVirtualFactCubeSimpleQuery() throws Exception {
-Configuration conf = getConf();
-conf.set(DRIVER_SUPPORTED_STORAGES, "C1");
-CubeQueryContext rewrittenQuery =
-  rewriteCtx("select SUM(msr2) from virtualCube where " + TWO_DAYS_RANGE, 
getConfWithStorages("C1"));
-String expected = getExpectedQuery(VIRTUAL_CUBE_NAME, "select 
sum(virtualcube.msr2) as `sum(msr2)` FROM ",
-  null, "AND ( dim1 = 10 )", 
getWhereForDailyAndHourly2days(VIRTUAL_CUBE_NAME,
-"C1_summary1"));
-String hql = rewrittenQuery.toHQL();
-compareQueries(hql, expected);
-System.out.println("Non existing parts:" + 
rewrittenQuery.getNonExistingParts());
-  }
-
-
   @Test
   public void testMaxCoveringFact() throws Exception {
 Configuration conf = getConf();
@@ -954,7 +938,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
   rewrite("select SUM(msr2) from testCube" + " where " + 
TWO_MONTHS_RANGE_UPTO_HOURS, getConfWithStorages("C2"));
 String expected =
   getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as 
`sum(msr2)` FROM ", null, null,
-getWhereForMonthlyDailyAndHourly2months("C2_testfact"));
+getWhereForMonthlyDailyAndHourly2months(TEST_CUBE_NAME, 
"C2_testfact"));
 compareQueries(hqlQuery, expected);
   }
 

[11/12] lens git commit: LENS-1451: Enforcing valid fact tables doesn't filter out segmentations

[prongs]

LENS-1451: Enforcing valid fact tables doesn't filter out segmentations


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3769ef0e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3769ef0e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3769ef0e

Branch: refs/heads/current-release-line
Commit: 3769ef0e8987caf26c931bf2735e3658e30f3ac4
Parents: 41051ea
Author: Rajat Khandelwal 
Authored: Thu Jul 6 19:35:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:43:00 2017 +0530

--
 .../lens/cube/parse/CandidateTableResolver.java | 26 ++--
 1 file changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/3769ef0e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index be3b474..86209bd 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -104,6 +104,9 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void populateCandidateTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
+  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
+  List validFactTables =
+StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
   List factTables = 
cubeql.getMetastoreClient().getAllFacts(cubeql.getCube());
   if (factTables.isEmpty()) {
 throw new 
LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(),
@@ -112,11 +115,15 @@ class CandidateTableResolver implements ContextRewriter {
   for (FactTable fact : factTables) {
 if (fact.getUpdatePeriods().isEmpty()) {
   log.info("Not considering fact: {} as it has no update periods", 
fact.getName());
+} else if (validFactTables != null && 
!validFactTables.contains(fact.getName())) {
+  log.info("Not considering fact: {} as it's not valid as per user 
configuration.", fact.getName());
 } else {
   for (String s : fact.getStorages()) {
 StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, 
s, cubeql);
 if (isStorageSupportedOnDriver(sc.getStorageName())) {
   cubeql.getCandidates().add(sc);
+} else {
+  log.info("Not considering {} since storage is not supported on 
driver.", sc.getName());
 }
   }
 }
@@ -125,7 +132,12 @@ class CandidateTableResolver implements ContextRewriter {
   log.info("Populated storage candidates: {}", cubeql.getCandidates());
   List segmentationCandidates = 
Lists.newArrayList();
   for (Segmentation segmentation : 
cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) {
-segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+if (validFactTables != null && 
!validFactTables.contains(segmentation.getName())) {
+  log.info("Not considering segmentation: {} as it's not valid as per 
user configuration.",
+segmentation.getName());
+} else {
+  segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+}
   }
   cubeql.getCandidates().addAll(segmentationCandidates);
 }
@@ -261,9 +273,6 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void resolveCandidateFactTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
-  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
-  List validFactTables =
-  StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
 
   Set queriedMsrs = new HashSet<>();
   Set dimExprs = new HashSet<>();
@@ -279,15 +288,6 @@ class CandidateTableResolver implements ContextRewriter {
 Candidate cand = i.next();
 if (cand instanceof StorageCandidate) {
   StorageCandidate sc = (StorageCandidate) cand;
-  if (validFactTables != null) {
-if 
(!validFactTables.contains(sc.getFact().getName().toLowerCase())) {
-  log.info("Not considering storage candidate:{} as it is not a 
valid candidate", sc);
-  cubeql.addStoragePruningMsg(sc, new 
CandidateTablePruneCause(CandidateTablePruneCode.INVALID));
-  

[07/12] lens git commit: LENS-1448: Having clause expressions are not resolved correctly for JoinCandidates

[prongs]

LENS-1448: Having clause expressions are not resolved correctly for 
JoinCandidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/54ab131a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/54ab131a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/54ab131a

Branch: refs/heads/current-release-line
Commit: 54ab131a2119a029dd8a470653d803fa0a7c35b6
Parents: fbad350
Author: Sushil Mohanty 
Authored: Wed Jun 28 13:10:55 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 13 14:42:55 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 11 +--
 .../lens/cube/parse/UnionQueryWriter.java   | 19 +++
 .../lens/cube/parse/TestBaseCubeQueries.java| 20 
 3 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 2403576..66b043e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -424,20 +424,19 @@ class ExpressionResolver implements ContextRewriter {
   }
   replaceAST(cubeql, queryAST.getJoinAST());
   replaceAST(cubeql, queryAST.getGroupByAST());
-  // Having AST is resolved by each fact, so that all facts can expand 
their expressions.
-  // Having ast is not copied now, it's maintained in cubeQueryContext, 
each fact processes that serially.
+  // Resolve having expression for StorageCandidate
   if (queryAST.getHavingAST() != null) {
 replaceAST(cubeql, queryAST.getHavingAST());
-  } else if (cubeql.getHavingAST() != null && 
nonPickedExpressionsForCandidate.isEmpty()) {
-replaceAST(cubeql, cubeql.getHavingAST());
-queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
+  } else if (cubeql.getHavingAST() != null) {
+ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+replaceAST(cubeql, havingCopy);
+queryAST.setHavingAST(havingCopy);
   }
   replaceAST(cubeql, queryAST.getOrderByAST());
 }
 
 private void replaceAST(final CubeQueryContext cubeql, ASTNode node) 
throws LensException {
   if (node == null) {
-
 return;
   }
   // Traverse the tree and resolve expression columns

http://git-wip-us.apache.org/repos/asf/lens/blob/54ab131a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f6c9ce1..9dc7ee6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -96,7 +96,6 @@ public class UnionQueryWriter extends SimpleHQLContext {
   if (sc.getQueryAst().getHavingAST() != null) {
 cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
   }
-  sc.getQueryAst().setHavingAST(null);
   sc.getQueryAst().setOrderByAST(null);
   sc.getQueryAst().setLimitValue(null);
 }
@@ -120,8 +119,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
   private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
-if (cubeql.getHavingAST() != null) {
-  ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+if (sc.getQueryAst().getHavingAST() == null
+&& cubeql.getHavingAST() != null) {
+  sc.getQueryAst().setHavingAST(cubeql.getHavingAST());
+}
+if (sc.getQueryAst().getHavingAST() != null) {
+  ASTNode havingCopy = 
MetastoreUtil.copyAST(sc.getQueryAst().getHavingAST());
   Set havingAggChildrenASTs = new LinkedHashSet<>();
   getAggregateChildrenInNode(havingCopy, havingAggChildrenASTs);
   processHavingExpression(innerAst, havingAggChildrenASTs, aliasDecider, 
sc);
@@ -283,8 +286,9 @@ public class UnionQueryWriter extends SimpleHQLContext {
   }
 
   private boolean isNodeDefault(ASTNode node) {
-if (HQLParser.isAggregateAST((ASTNode) node.getChild(0))) {
-  if (HQLParser.getString((ASTNode) 
node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)) {
+if (HQLParser.isAggregateAST((ASTNode) node.getChild(0)) || 

lens git commit: LENS-1452: Optimize Time Union candidate Algorithm

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 45521bf2c -> d4236668c


LENS-1452: Optimize Time Union candidate Algorithm


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d4236668
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d4236668
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d4236668

Branch: refs/heads/master
Commit: d4236668c3eb3a8bded92046574618d5d2e38bcf
Parents: 45521bf
Author: Rajat Khandelwal 
Authored: Mon Jul 10 16:52:54 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Jul 10 16:52:54 2017 +0530

--
 .../parse/CandidateCoveringSetsResolver.java| 47 +---
 1 file changed, 41 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d4236668/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 8e07162..69d9562 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -26,6 +26,7 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.TimeRange;
 import org.apache.lens.server.api.error.LensException;
 
+import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -124,16 +125,14 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
 // Get all covering fact sets
-List unionCoveringSet =
-getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
+//List unionCoveringSet = getCombinations(new 
ArrayList<>(allCandidatesPartiallyValid), cubeql);
+List unionCoveringSet = 
getCombinationTailIterative(allCandidatesPartiallyValid, cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
 }
-// prune redundant covering sets
-pruneRedundantUnionCoveringSets(unionCoveringSet);
 // pruing done in the previous steps, now create union candidates
 candidateSet.addAll(unionCoveringSet);
 updateQueriableMeasures(candidateSet, qpcList);
@@ -155,7 +154,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private void pruneRedundantUnionCoveringSets(List 
candidates) {
 for (int i = 0; i < candidates.size(); i++) {
   UnionCandidate current = candidates.get(i);
@@ -168,7 +167,7 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
   }
 }
   }
-
+  @Deprecated
   private List getCombinations(final List 
candidates, CubeQueryContext cubeql) {
 List combinations = new LinkedList<>();
 int size = candidates.size();
@@ -193,6 +192,42 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return combinations;
   }
 
+  /**
+   * The following function is iterative rewrite of the following 
tail-recursive implementation:
+   * (ignoring cubeql for clarity)
+   * getCombinations(candidates) = getCombinationsTailRecursive(emptyList(), 
candidates)
+   *
+   * getCombinationsTailRecursive(incompleteCombinations: 
List[List[Candidate]], candidates: List[Candidate]) =
+   *   head, tail = head and tail of linked List candidates
+   *   add head to all elements of incompleteCombinations.
+   *   complete = remove now complete combinations from incompleteCombinations
+   *   return complete ++ getCombinationsTailRecursive(incompleteCombinations, 
tail)
+   * @param candidates
+   * @param cubeql
+   * @return
+   */
+  private List getCombinationTailIterative(List 
candidates, CubeQueryContext cubeql) {
+LinkedList candidateLinkedList = 
Lists.newLinkedList(candidates);
+List incompleteCombinations = Lists.newArrayList();
+List unionCandidates = Lists.newArrayList();
+
+while(!candidateLinkedList.isEmpty()) {
+  Candidate candidate = candidateLinkedList.remove();
+  incompleteCombinations.add(Lists.newArrayList());
+  Iterator iter = incompleteCombinations.iterator();
+  while(iter.hasNext()) {
+List incompleteCombination = iter.next();
+incompleteCombination.add(candidate);
+UnionCandidate unionCandidate = new 

lens git commit: LENS-1451: Enforcing valid fact tables doesn't filter out segmentations

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master e570e9e33 -> 45521bf2c


LENS-1451: Enforcing valid fact tables doesn't filter out segmentations


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/45521bf2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/45521bf2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/45521bf2

Branch: refs/heads/master
Commit: 45521bf2c05b2babb2fc96a2000c3dc11fb4329c
Parents: e570e9e
Author: Rajat Khandelwal 
Authored: Thu Jul 6 19:35:15 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jul 6 19:35:15 2017 +0530

--
 .../lens/cube/parse/CandidateTableResolver.java | 26 ++--
 1 file changed, 13 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/45521bf2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index be3b474..86209bd 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -104,6 +104,9 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void populateCandidateTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
+  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
+  List validFactTables =
+StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
   List factTables = 
cubeql.getMetastoreClient().getAllFacts(cubeql.getCube());
   if (factTables.isEmpty()) {
 throw new 
LensException(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(),
@@ -112,11 +115,15 @@ class CandidateTableResolver implements ContextRewriter {
   for (FactTable fact : factTables) {
 if (fact.getUpdatePeriods().isEmpty()) {
   log.info("Not considering fact: {} as it has no update periods", 
fact.getName());
+} else if (validFactTables != null && 
!validFactTables.contains(fact.getName())) {
+  log.info("Not considering fact: {} as it's not valid as per user 
configuration.", fact.getName());
 } else {
   for (String s : fact.getStorages()) {
 StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, 
s, cubeql);
 if (isStorageSupportedOnDriver(sc.getStorageName())) {
   cubeql.getCandidates().add(sc);
+} else {
+  log.info("Not considering {} since storage is not supported on 
driver.", sc.getName());
 }
   }
 }
@@ -125,7 +132,12 @@ class CandidateTableResolver implements ContextRewriter {
   log.info("Populated storage candidates: {}", cubeql.getCandidates());
   List segmentationCandidates = 
Lists.newArrayList();
   for (Segmentation segmentation : 
cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) {
-segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+if (validFactTables != null && 
!validFactTables.contains(segmentation.getName())) {
+  log.info("Not considering segmentation: {} as it's not valid as per 
user configuration.",
+segmentation.getName());
+} else {
+  segmentationCandidates.add(new SegmentationCandidate(cubeql, 
segmentation));
+}
   }
   cubeql.getCandidates().addAll(segmentationCandidates);
 }
@@ -261,9 +273,6 @@ class CandidateTableResolver implements ContextRewriter {
 
   private void resolveCandidateFactTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
-  String str = 
cubeql.getConf().get(CubeQueryConfUtil.getValidFactTablesKey(cubeql.getCube().getName()));
-  List validFactTables =
-  StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
 
   Set queriedMsrs = new HashSet<>();
   Set dimExprs = new HashSet<>();
@@ -279,15 +288,6 @@ class CandidateTableResolver implements ContextRewriter {
 Candidate cand = i.next();
 if (cand instanceof StorageCandidate) {
   StorageCandidate sc = (StorageCandidate) cand;
-  if (validFactTables != null) {
-if 
(!validFactTables.contains(sc.getFact().getName().toLowerCase())) {
-  log.info("Not considering storage candidate:{} as it is not a 
valid candidate", sc);
-  cubeql.addStoragePruningMsg(sc, 

lens git commit: LENS-1450: filters are getting replicated in jdbc driver query

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 01854d38b -> 63e0b6917


LENS-1450: filters are getting replicated in jdbc driver query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/63e0b691
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/63e0b691
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/63e0b691

Branch: refs/heads/master
Commit: 63e0b691741b9ac045cbfe6dd988000988077800
Parents: 01854d3
Author: Rajitha R 
Authored: Wed Jul 5 16:49:07 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Jul 5 16:49:07 2017 +0530

--
 .../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/63e0b691/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b5330a7..fb49b6d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1169,7 +1169,7 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 }
   }
 } else if (node.getParent() == null
-&& node.getToken().getType() != HiveParser.KW_AND) {
+&& node.getToken().getType() != HiveParser.KW_AND && 
node.getChildCount() == 0) {
   // if node is the only child
   allFilters.add(HQLParser.getString((ASTNode) node));
 }

lens git commit: LENS-1446: Cube latestdate api broken after virtual fact

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master e2d6cbb94 -> 01854d38b


LENS-1446: Cube latestdate api broken after virtual fact


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/01854d38
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/01854d38
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/01854d38

Branch: refs/heads/master
Commit: 01854d38ba4e2c1502008d2f388d00fcf5836080
Parents: e2d6cbb
Author: Rajitha R 
Authored: Thu Jun 29 17:23:26 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu Jun 29 17:23:26 2017 +0530

--
 .../apache/lens/cli/commands/LensSchemaCommands.java |  2 +-
 .../lens/cube/metadata/CubeMetastoreClient.java  | 15 ---
 .../org/apache/lens/cube/parse/StorageCandidate.java |  8 
 3 files changed, 13 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/01854d38/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
--
diff --git 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index aca1cf9..cc184a8 100644
--- 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -113,7 +113,7 @@ public class LensSchemaCommands implements CommandMarker {
 CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s");
 UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s 
--path %s");
 CREATE_COMMAND_MAP.put(XVirtualFactTable.class, "create fact --path %s");
-UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --name %s 
--path %s");
+UPDATE_COMMAND_MAP.put(XVirtualFactTable.class, "update fact --fact_name 
%s --path %s");
 CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path 
%s");
 UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s 
--path %s");
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/01854d38/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 5c05e53..3952696 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -161,7 +161,8 @@ public class CubeMetastoreClient {
 for (FactTable fact : getAllFacts(cube)) {
   for (String storage : fact.getStorages()) {
 for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) 
{
-  PartitionTimeline timeline = 
partitionTimelineCache.get(fact.getName(), storage, updatePeriod, partCol);
+  PartitionTimeline timeline = 
partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod,
+partCol);
   if (timeline != null) {// this storage table is partitioned by 
partCol or not.
 Date latest = timeline.getLatestDate();
 if (latest != null && latest.after(max)) {
@@ -1078,8 +1079,8 @@ public class CubeMetastoreClient {
   List partsAdded = new ArrayList<>();
   // first update in memory, then add to hive table's partitions. delete 
is reverse.
   partitionTimelineCache.updateForAddition(factOrDimTable, storageName, 
updatePeriod,
-  getTimePartSpecs(storagePartitionDescs, 
getStorageTableStartDate(storageTableName, factOrDimTable),
-  getStorageTableEndDate(storageTableName, 
factOrDimTable)));
+  getTimePartSpecs(storagePartitionDescs, 
getStorageTableStartDate(storageTableName,
+getFactTable(factOrDimTable)), 
getStorageTableEndDate(storageTableName, getFactTable(factOrDimTable;
   // Adding partition in fact table.
   if (storagePartitionDescs.size() > 0) {
 partsAdded = getStorage(storageName).addPartitions(getClient(), 
factOrDimTable, updatePeriod,
@@ -1094,17 +1095,17 @@ public class CubeMetastoreClient {
 }
   }
 
-  public Date getStorageTableStartDate(String storageTable, String 
factTableName)
+  public Date getStorageTableStartDate(String storageTable, FactTable 
factTableName)
 throws LensException {
 List startDates = getStorageTimes(storageTable, 
MetastoreUtil.getStoragetableStartTimesKey());
-startDates.add(getFactTable(factTableName).getStartTime());
+startDates.add(factTableName.getStartTime());
 

lens git commit: LENS-1448: Having clause expressions are not resolved correctly for JoinCandidates

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 67adf5b84 -> e2d6cbb94


LENS-1448: Having clause expressions are not resolved correctly for 
JoinCandidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e2d6cbb9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e2d6cbb9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e2d6cbb9

Branch: refs/heads/master
Commit: e2d6cbb947441e7c3c2a425cc9cf25ffadccf08b
Parents: 67adf5b
Author: Sushil Mohanty 
Authored: Wed Jun 28 13:10:55 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Jun 28 13:10:55 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 11 +--
 .../lens/cube/parse/UnionQueryWriter.java   | 19 +++
 .../lens/cube/parse/TestBaseCubeQueries.java| 20 
 3 files changed, 36 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/e2d6cbb9/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 2403576..66b043e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -424,20 +424,19 @@ class ExpressionResolver implements ContextRewriter {
   }
   replaceAST(cubeql, queryAST.getJoinAST());
   replaceAST(cubeql, queryAST.getGroupByAST());
-  // Having AST is resolved by each fact, so that all facts can expand 
their expressions.
-  // Having ast is not copied now, it's maintained in cubeQueryContext, 
each fact processes that serially.
+  // Resolve having expression for StorageCandidate
   if (queryAST.getHavingAST() != null) {
 replaceAST(cubeql, queryAST.getHavingAST());
-  } else if (cubeql.getHavingAST() != null && 
nonPickedExpressionsForCandidate.isEmpty()) {
-replaceAST(cubeql, cubeql.getHavingAST());
-queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
+  } else if (cubeql.getHavingAST() != null) {
+ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+replaceAST(cubeql, havingCopy);
+queryAST.setHavingAST(havingCopy);
   }
   replaceAST(cubeql, queryAST.getOrderByAST());
 }
 
 private void replaceAST(final CubeQueryContext cubeql, ASTNode node) 
throws LensException {
   if (node == null) {
-
 return;
   }
   // Traverse the tree and resolve expression columns

http://git-wip-us.apache.org/repos/asf/lens/blob/e2d6cbb9/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f6c9ce1..9dc7ee6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -96,7 +96,6 @@ public class UnionQueryWriter extends SimpleHQLContext {
   if (sc.getQueryAst().getHavingAST() != null) {
 cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
   }
-  sc.getQueryAst().setHavingAST(null);
   sc.getQueryAst().setOrderByAST(null);
   sc.getQueryAst().setLimitValue(null);
 }
@@ -120,8 +119,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
*/
   private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
-if (cubeql.getHavingAST() != null) {
-  ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
+if (sc.getQueryAst().getHavingAST() == null
+&& cubeql.getHavingAST() != null) {
+  sc.getQueryAst().setHavingAST(cubeql.getHavingAST());
+}
+if (sc.getQueryAst().getHavingAST() != null) {
+  ASTNode havingCopy = 
MetastoreUtil.copyAST(sc.getQueryAst().getHavingAST());
   Set havingAggChildrenASTs = new LinkedHashSet<>();
   getAggregateChildrenInNode(havingCopy, havingAggChildrenASTs);
   processHavingExpression(innerAst, havingAggChildrenASTs, aliasDecider, 
sc);
@@ -283,8 +286,9 @@ public class UnionQueryWriter extends SimpleHQLContext {
   }
 
   private boolean isNodeDefault(ASTNode node) {
-if (HQLParser.isAggregateAST((ASTNode) node.getChild(0))) {
-  if (HQLParser.getString((ASTNode) 
node.getChild(0).getChild(1)).equals(DEFAULT_MEASURE)) {

[2/2] lens git commit: LENS-1445: Expression having reference column ends up rewriting wrong query

[prongs]

LENS-1445: Expression having reference column ends up rewriting wrong query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/70722824
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/70722824
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/70722824

Branch: refs/heads/master
Commit: 70722824b664aed06df24b0fe2d8b46f1f9d0e88
Parents: 64448ed
Author: Sushil Mohanty 
Authored: Fri Jun 23 16:40:25 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri Jun 23 16:40:25 2017 +0530

--
 .../lens/cube/parse/UnionQueryWriter.java   | 79 +---
 .../parse/TestCubeSegmentationRewriter.java |  3 +-
 .../cube/parse/TestUnionAndJoinCandidates.java  | 26 +++
 .../resources/schema/cubes/base/basecube.xml| 11 +++
 .../resources/schema/cubes/base/testcube.xml|  1 +
 5 files changed, 92 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/70722824/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index cc0a2e5..f6c9ce1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -28,7 +28,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
 import java.util.*;
 
-import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.JoinPath;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.hive.ql.lib.Node;
@@ -117,7 +118,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
* @return ASTNode
* @throws LensException
*/
-  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidate sc)
+  private ASTNode processHavingAST(ASTNode innerAst, AliasDecider 
aliasDecider, StorageCandidateHQLContext sc)
 throws LensException {
 if (cubeql.getHavingAST() != null) {
   ASTNode havingCopy = MetastoreUtil.copyAST(cubeql.getHavingAST());
@@ -251,25 +252,15 @@ public class UnionQueryWriter extends SimpleHQLContext {
 for (StorageCandidateHQLContext sc : storageCandidates) {
   node = (ASTNode) 
sc.getQueryAst().getSelectAST().getChild(position).getChild(0);
   if (HQLParser.isAggregateAST(node) || HQLParser.hasAggregate(node)) {
-return MetastoreUtil.copyAST(node);
+if (!node.getChild(1).toString().equals(DEFAULT_MEASURE))  {
+  return MetastoreUtil.copyAST(node);
+}
   }
 }
 return MetastoreUtil.copyAST(node);
   }
 
   /**
-   *  Check if ASTNode is answerable by StorageCandidate
-   * @param sc
-   * @param node
-   * @return
-   */
-  private boolean isNodeNotAnswerableForStorageCandidate(StorageCandidate sc, 
ASTNode node) {
-Set cols = new LinkedHashSet<>();
-getAllColumnsOfNode(node, cols);
-return !sc.getColumns().containsAll(cols);
-  }
-
-  /**
* Set the default value "0.0" in the non answerable aggreagte expressions.
* @param node
* @param sc
@@ -467,7 +458,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
 // Iterate over the StorageCandidates and add non projected having columns 
in inner select ASTs
 for (StorageCandidateHQLContext sc : storageCandidates) {
   aliasDecider.setCounter(selectAliasCounter);
-  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, 
sc.getStorageCandidate());
+  processHavingAST(sc.getQueryAst().getSelectAST(), aliasDecider, sc);
 }
 removeRedundantProjectedPhrases();
   }
@@ -493,7 +484,7 @@ public class UnionQueryWriter extends SimpleHQLContext {
   ASTNode child = (ASTNode) selectAST.getChild(i);
   ASTNode outerSelect = new ASTNode(child);
   ASTNode selectExprAST = (ASTNode) child.getChild(0);
-  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc.getStorageCandidate(), true,
+  ASTNode outerAST = getOuterAST(selectExprAST, innerSelectAST, 
aliasDecider, sc, true,
   cubeql.getBaseCube().getDimAttributeNames());
   outerSelect.addChild(outerAST);
   // has an alias? add it
@@ -529,13 +520,14 @@ public class UnionQueryWriter extends SimpleHQLContext {
5. If given ast is memorized as mentioned in the above cases, return the 
mapping.
  */
   private ASTNode getOuterAST(ASTNode astNode, ASTNode innerSelectAST,
-  AliasDecider aliasDecider, StorageCandidate sc, boolean isSelectAst, 
Set dimensionSet)
+  AliasDecider 

[1/2] lens git commit: LENS-1444: Optimize the algorithm of finding all eligible union candidates

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 2f5be8139 -> 70722824b


LENS-1444: Optimize the algorithm of finding all eligible union candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/64448ed6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/64448ed6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/64448ed6

Branch: refs/heads/master
Commit: 64448ed6049f0536f8c8284628e808ec5843e173
Parents: 2f5be81
Author: Rajat Khandelwal 
Authored: Fri Jun 23 16:39:37 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri Jun 23 16:39:37 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java  | 17 -
 1 file changed, 4 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/64448ed6/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 61c28c6..8e07162 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -106,16 +106,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 return true;
   }
 
-  private void pruneUnionCandidatesNotCoveringAllRanges(List 
ucs, CubeQueryContext cubeql) {
-for (Iterator itr = ucs.iterator(); itr.hasNext();) {
-  UnionCandidate uc = itr.next();
-  if (!isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
-itr.remove();
-cubeql.addCandidatePruningMsg(uc, 
CandidateTablePruneCause.storageNotAvailableInRange(cubeql.getTimeRanges()));
-  }
-}
-  }
-
   private List resolveTimeRangeCoveringFactSet(CubeQueryContext 
cubeql,
   Set queriedMsrs, List 
qpcList) throws LensException {
 List candidateSet = new ArrayList<>();
@@ -138,8 +128,6 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql);
 // Sort the Collection based on no of elements
 unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount));
-// prune non covering sets
-pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql);
 // prune candidate set which doesn't contain any common measure i
 if (!queriedMsrs.isEmpty()) {
   pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
@@ -197,7 +185,10 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 clonedI = clonedI >>> 1;
 --count;
   }
-  combinations.add(new UnionCandidate(individualCombinationList, cubeql));
+  UnionCandidate uc = new UnionCandidate(individualCombinationList, 
cubeql);
+  if (isCandidateCoveringTimeRanges(uc, cubeql.getTimeRanges())) {
+combinations.add(uc);
+  }
 }
 return combinations;
   }

lens git commit: LENS-1443: Fallback ranges not working for virtual facts

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 6f86e785e -> 2f5be8139


LENS-1443: Fallback ranges not working for virtual facts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2f5be813
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2f5be813
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2f5be813

Branch: refs/heads/master
Commit: 2f5be81397e9212bec3845bfece596e2079d8a18
Parents: 6f86e78
Author: Rajat Khandelwal 
Authored: Wed Jun 21 13:20:09 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Jun 21 13:20:09 2017 +0530

--
 .../apache/lens/cube/metadata/FactTable.java| 38 ++--
 .../apache/lens/cube/parse/JoinCandidate.java   |  7 ++--
 .../lens/cube/parse/LeastPartitionResolver.java | 21 +--
 .../lens/cube/parse/StorageCandidate.java   |  2 +-
 .../lens/cube/parse/StorageTableResolver.java   |  5 +--
 5 files changed, 37 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/2f5be813/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index f87cf44..a463c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -35,132 +35,132 @@ public interface FactTable extends Named {
*
* @return Map of storage to set of update periods
*/
-  public Map getUpdatePeriods();
+  Map getUpdatePeriods();
 
   /**
* Cube to which this fact belongs to
*
* @return the cube string
*/
-  public String getCubeName();
+  String getCubeName();
 
   /**
* The set of Storage names
*
* @return set of strings
*/
-  public Set getStorages();
+  Set getStorages();
 
   /**
*The type of the fact
*
* @return table type {@link CubeTableType}
*/
-  public CubeTableType getTableType();
+  CubeTableType getTableType();
 
   /**
* Config properties
*
* @return map of string, string
*/
-  public Map getProperties();
+  Map getProperties();
 
   /**
* Valid columns of the fact
*
* @return list of column names
*/
-  public Set getValidColumns();
+  Set getValidColumns();
 
   /**
* Weight of the fact
*
* @return weight of the fact in double
*/
-  public double weight();
+  double weight();
 
   /**
* Set of all the columns names of the fact
*
* @return set of column names
*/
-  public Set getAllFieldNames();
+  Set getAllFieldNames();
 
   /**
*tag for checking data completeness
*
* @return Tag String
*/
-  public String getDataCompletenessTag();
+  String getDataCompletenessTag();
 
   /**
* List of columns of the fact
*
* @return set of {@link FieldSchema}
*/
-  public List getColumns();
+  List getColumns();
 
   /**
* Is Aggregated Fact
*
* @return true if fact is Aggregated , false otherwise
*/
-  public boolean isAggregated();
+  boolean isAggregated();
 
   /**
* Absolute start time of the fact
*
* @return Absolute Start time of the fact {@link Date}
*/
-  public Date getAbsoluteStartTime();
+  Date getAbsoluteStartTime();
 
   /**
* Relative start time of the fact
*
* @return Relative Start time of the fact {@link Date}
*/
-  public Date getRelativeStartTime();
+  Date getRelativeStartTime();
 
   /**
* Start time of the fact
*
* @return Start time of the fact {@link Date}
*/
-  public Date getStartTime();
+  Date getStartTime();
 
   /**
* Absolute end time of the fact
*
* @return Absolute End time of the fact {@link Date}
*/
-  public Date getAbsoluteEndTime();
+  Date getAbsoluteEndTime();
 
   /**
* Relative End time of the Fact
*
* @return Relative end time of the fact {@link Date}
*/
-  public Date getRelativeEndTime();
+  Date getRelativeEndTime();
 
   /**
* End time of the fact
*
* @return End time of the fact {@link Date}
*/
-  public Date getEndTime();
+  Date getEndTime();
 
   /**
* Is Virtual Fact
*
* @return true if fact is a virtual fact, false otherwise
*/
-  public boolean isVirtualFact();
+  boolean isVirtualFact();
 
   /**
* Storage name of the fact
*
* @return Storage name of the fact
*/
-  public String getSourceFactName();
+  String getSourceFactName();
 
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/2f5be813/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java

lens git commit: LENS-1442: Optimize algorithm of CandidateCoveringSetResolver

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 8a7478d46 -> 6f86e785e


LENS-1442: Optimize algorithm of CandidateCoveringSetResolver


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6f86e785
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6f86e785
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6f86e785

Branch: refs/heads/master
Commit: 6f86e785ea79cc8a3c03ceba282a0cfadb3ae3c6
Parents: 8a7478d
Author: Rajat Khandelwal 
Authored: Tue Jun 20 16:09:32 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 20 16:09:32 2017 +0530

--
 .../cube/parse/CandidateCoveringSetsResolver.java| 15 +--
 1 file changed, 9 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/6f86e785/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 1e9873f..61c28c6 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -208,21 +208,24 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 List ucSet = new ArrayList<>(candidates);
 // Check if a single set can answer all the measures and exprsWithMeasures
 for (Iterator i = ucSet.iterator(); i.hasNext();) {
-  boolean evaluable = false;
+  boolean allEvaluable = true;
+  boolean anyEvaluable = false;
   Candidate uc = i.next();
   for (QueriedPhraseContext msr : msrs) {
-evaluable = uc.isPhraseAnswerable(msr);
-if (!evaluable) {
-  break;
-}
+boolean evaluable = uc.isPhraseAnswerable(msr);
+allEvaluable &= evaluable;
+anyEvaluable |= evaluable;
   }
-  if (evaluable) {
+  if (allEvaluable) {
 // single set can answer all the measures as an UnionCandidate
 List one = new ArrayList<>();
 one.add(uc);
 msrCoveringSets.add(one);
 i.remove();
   }
+  if (!anyEvaluable) { // none evaluable
+i.remove();
+  }
 }
 // Sets that contain all measures or no measures are removed from 
iteration.
 // find other facts

lens git commit: LENS-1441: CandidateTableResolver should not add StorageCandidate if storage is not supported

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 2f56c741b -> 8a7478d46


LENS-1441: CandidateTableResolver should not add StorageCandidate if storage is 
not supported


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8a7478d4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8a7478d4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8a7478d4

Branch: refs/heads/master
Commit: 8a7478d468e40749627737cbe5a7db7736b2c291
Parents: 2f56c74
Author: Rajat Khandelwal 
Authored: Tue Jun 20 16:08:00 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 20 16:08:00 2017 +0530

--
 .../lens/cube/parse/CandidateTableResolver.java | 24 +++-
 .../lens/cube/parse/CubeQueryRewriter.java  |  2 +-
 .../lens/cube/parse/ExpressionResolver.java |  5 +++-
 .../lens/cube/parse/StorageTableResolver.java   |  6 -
 .../cube/parse/TestDenormalizationResolver.java | 10 ++--
 .../lens/cube/parse/TestTimeRangeResolver.java  | 20 +++-
 .../server/query/QueryAPIErrorResponseTest.java |  1 +
 7 files changed, 39 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/8a7478d4/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index f530650..be3b474 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -29,6 +29,7 @@ import 
org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -48,8 +49,15 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 class CandidateTableResolver implements ContextRewriter {
 
+  private final List supportedStorages;
+  private final boolean allStoragesSupported;
   private boolean checkForQueriedColumns = true;
 
+  public CandidateTableResolver(Configuration conf) {
+this.supportedStorages = getSupportedStorages(conf);
+this.allStoragesSupported = (supportedStorages == null);
+  }
+
   @Override
   public void rewriteContext(CubeQueryContext cubeql) throws LensException {
 if (checkForQueriedColumns) {
@@ -81,6 +89,18 @@ class CandidateTableResolver implements ContextRewriter {
   checkForQueriedColumns = true;
 }
   }
+  private List getSupportedStorages(Configuration conf) {
+String[] storages = 
conf.getStrings(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES);
+if (storages != null) {
+  return Arrays.asList(storages);
+}
+return null;
+  }
+
+  private boolean isStorageSupportedOnDriver(String storage) {
+return allStoragesSupported || supportedStorages.contains(storage);
+  }
+
 
   private void populateCandidateTables(CubeQueryContext cubeql) throws 
LensException {
 if (cubeql.getCube() != null) {
@@ -95,7 +115,9 @@ class CandidateTableResolver implements ContextRewriter {
 } else {
   for (String s : fact.getStorages()) {
 StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, 
s, cubeql);
-cubeql.getCandidates().add(sc);
+if (isStorageSupportedOnDriver(sc.getStorageName())) {
+  cubeql.getCandidates().add(sc);
+}
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/8a7478d4/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index 0ef41f3..143b266 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -147,7 +147,7 @@ public class CubeQueryRewriter {
 rewriters.add(new AliasReplacer());
 ExpressionResolver exprResolver = new ExpressionResolver();
 DenormalizationResolver denormResolver = new DenormalizationResolver();
-CandidateTableResolver candidateTblResolver = new CandidateTableResolver();
+CandidateTableResolver candidateTblResolver = new 
CandidateTableResolver(conf);
 StorageTableResolver storageTableResolver = new StorageTableResolver(conf);
 LightestFactResolver 

[2/2] lens git commit: LENS-1439: Having clause getting skipped if query goes to a segmentation containing single segment

[prongs]

LENS-1439: Having clause getting skipped if query goes to a segmentation 
containing single segment


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2f56c741
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2f56c741
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2f56c741

Branch: refs/heads/master
Commit: 2f56c741bffea20678007eff4e0173bb6dc06110
Parents: 4ecc595
Author: Rajat Khandelwal 
Authored: Fri Jun 16 14:53:41 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri Jun 16 14:53:41 2017 +0530

--
 .../org/apache/lens/cube/parse/StorageCandidateHQLContext.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/2f56c741/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
index cca39c0..494b08e 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidateHQLContext.java
@@ -50,7 +50,7 @@ public class StorageCandidateHQLContext extends DimHQLContext 
{
 this.storageCandidate = storageCandidate;
 this.rootCubeQueryContext = rootCubeQueryContext;
 getCubeQueryContext().addRangeClauses(this);
-if (!isRoot()) {
+if (!Objects.equals(getStorageCandidate(), 
rootCubeQueryContext.getPickedCandidate())) {
   getQueryAst().setHavingAST(null);
 }
   }

[1/2] lens git commit: LENS-1438: Testcase failure after LENS-1432

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 191756ade -> 2f56c741b


LENS-1438: Testcase failure after LENS-1432


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4ecc5957
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4ecc5957
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4ecc5957

Branch: refs/heads/master
Commit: 4ecc59576927a771d1c0c5789fbde1fc44fbce05
Parents: 191756a
Author: Rajat Khandelwal 
Authored: Fri Jun 16 14:52:57 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri Jun 16 14:52:57 2017 +0530

--
 .../lens/server/rewrite/TestRewriting.java  | 31 ++--
 1 file changed, 15 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/4ecc5957/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java 
b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
index 83dd1f7..304d21e 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
@@ -25,6 +25,7 @@ import org.apache.lens.cube.metadata.CubeMetastoreClient;
 import org.apache.lens.cube.parse.CubeQueryContext;
 import org.apache.lens.cube.parse.CubeQueryRewriter;
 import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.cube.parse.QueryWriterContext;
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.driver.LensDriver;
 import org.apache.lens.server.api.driver.MockDriver;
@@ -55,6 +56,7 @@ import org.testng.annotations.ObjectFactory;
 import org.testng.annotations.Test;
 
 import com.codahale.metrics.MetricRegistry;
+import com.google.common.collect.Maps;
 
 /**
  * The Class TestRewriting.
@@ -96,24 +98,18 @@ public class TestRewriting {
 
   private CubeQueryRewriter getMockedRewriter() throws ParseException, 
LensException, HiveException {
 CubeQueryRewriter mockwriter = Mockito.mock(CubeQueryRewriter.class);
-
Mockito.when(mockwriter.rewrite(Matchers.any(String.class))).thenAnswer(new 
Answer() {
-  @Override
-  public CubeQueryContext answer(InvocationOnMock invocation) throws 
Throwable {
-Object[] args = invocation.getArguments();
-// return query for first NUM_SUCCESS calls and fail later
-if (++i <= NUM_SUCCESS) {
-  return getMockedCubeContext((String) args[0]);
-} else {
-  throw new RuntimeException("Mock fail");
-}
+
Mockito.when(mockwriter.rewrite(Matchers.any(String.class))).thenAnswer(invocation
 -> {
+  Object[] args = invocation.getArguments();
+  // return query for first NUM_SUCCESS calls and fail later
+  if (++i <= NUM_SUCCESS) {
+return getMockedCubeContext((String) args[0]);
+  } else {
+throw new RuntimeException("Mock fail");
   }
 });
-
Mockito.when(mockwriter.rewrite(Matchers.any(ASTNode.class))).thenAnswer(new 
Answer() {
-  @Override
-  public CubeQueryContext answer(InvocationOnMock invocation) throws 
Throwable {
-Object[] args = invocation.getArguments();
-return getMockedCubeContext((ASTNode) args[0]);
-  }
+
Mockito.when(mockwriter.rewrite(Matchers.any(ASTNode.class))).thenAnswer(invocation
 -> {
+  Object[] args = invocation.getArguments();
+  return getMockedCubeContext((ASTNode) args[0]);
 });
 Mockito.doCallRealMethod().when(mockwriter).clear();
 return mockwriter;
@@ -130,6 +126,9 @@ public class TestRewriting {
   private CubeQueryContext getMockedCubeContext(String query)
 throws ParseException, LensException {
 CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
+QueryWriterContext mockQueryWriterContext = 
Mockito.mock(QueryWriterContext.class);
+
Mockito.when(mockQueryWriterContext.getDimsToQuery()).thenReturn(Maps.newHashMap());
+
Mockito.when(context.getQueryWriterContext()).thenReturn(mockQueryWriterContext);
 Mockito.when(context.toHQL()).thenReturn(query.substring(4));
 Mockito.when(context.toAST(Matchers.any(Context.class)))
   .thenReturn(HQLParser.parseHQL(query.toLowerCase().replaceFirst("^cube", 
""), hconf));

lens git commit: LENS-1433: Virtual Fact over a fact with single storage - multi update period is writing wrong query

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 949974d60 -> 191756ade


LENS-1433: Virtual Fact over a fact with single storage - multi update period 
is writing wrong query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/191756ad
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/191756ad
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/191756ad

Branch: refs/heads/master
Commit: 191756adec2ebe0147cdd7a3bff8e7f6584e7354
Parents: 949974d
Author: Rajitha R 
Authored: Wed Jun 14 15:06:56 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Jun 14 15:06:56 2017 +0530

--
 .../java/org/apache/lens/cube/parse/StorageCandidate.java| 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/191756ad/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index c8ff3b8..6e5aa4c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -265,7 +265,7 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
   Set uniqueStorageTables = new HashSet<>();
   for (UpdatePeriod updatePeriod : validUpdatePeriods) {
 uniqueStorageTables.add(
-  getCubeMetastoreClient().getStorageTableName(fact.getName(), 
storageName, updatePeriod)
+  
getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), 
storageName, updatePeriod)
 );
   }
   return uniqueStorageTables;
@@ -891,7 +891,7 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
   return this.startTime;
 }
 return getCubeMetastoreClient().getStorageTableStartDate(
-  getCubeMetastoreClient().getStorageTableName(fact.getName(), 
storageName, interval), fact.getName());
+  getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), 
storageName, interval), fact.getName());
   }
 
   private Date getStorageTableEndDate(UpdatePeriod interval) throws 
LensException {
@@ -900,7 +900,7 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
   return this.endTime;
 }
 return getCubeMetastoreClient().getStorageTableEndDate(
-  getCubeMetastoreClient().getStorageTableName(fact.getName(), 
storageName, interval), fact.getName());
+  getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), 
storageName, interval), fact.getName());
   }
 
 
@@ -931,7 +931,7 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
   StorageCandidate updatePeriodSpecificSc;
   for (UpdatePeriod period : participatingUpdatePeriods) {
 updatePeriodSpecificSc = copy();
-
updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getName(),
+
updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(),
   storageName, period));
 updatePeriodSpecificSc.truncatePartitions(period);
 periodSpecificScList.add(updatePeriodSpecificSc);

lens git commit: LENS-1432 : Map Join tuning broken for this query leading to Map timeouts

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 359937fab -> 933cc3d6f


LENS-1432 : Map Join tuning broken for this query leading to Map timeouts


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/933cc3d6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/933cc3d6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/933cc3d6

Branch: refs/heads/master
Commit: 933cc3d6f7924c893f1c18273ffba8347bfcec89
Parents: 359937f
Author: Sushil Mohanty 
Authored: Tue Jun 6 18:14:39 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 18:14:39 2017 +0530

--
 .../cube/parse/MultiCandidateQueryWriterContext.java | 11 +++
 .../org/apache/lens/cube/parse/QueryWriterContext.java   |  4 
 .../java/org/apache/lens/driver/cube/RewriterPlan.java   |  6 +++---
 .../org/apache/lens/cube/parse/TestRewriterPlan.java |  2 ++
 .../lens/cube/parse/TestUnionAndJoinCandidates.java  |  2 +-
 lens-cube/src/test/resources/schema/facts/b1fact1.xml|  2 +-
 6 files changed, 22 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
index d57c027..50046a5 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiCandidateQueryWriterContext.java
@@ -18,8 +18,11 @@
  */
 package org.apache.lens.cube.parse;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
+import org.apache.lens.cube.metadata.Dimension;
 import org.apache.lens.server.api.error.LensException;
 
 import com.google.common.collect.Lists;
@@ -90,4 +93,12 @@ public class MultiCandidateQueryWriterContext implements 
QueryWriterContext {
 List leafWriterContexts = 
getLeafQueryWriterContexts();
 return new UnionQueryWriter(leafWriterContexts, getCubeQueryContext());
   }
+
+  public Map getDimsToQuery() {
+Map allDimsQueried = new HashMap<>();
+for (QueryWriterContext ctx : children) {
+  allDimsQueried.putAll(ctx.getDimsToQuery());
+}
+return allDimsQueried;
+  }
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
index d55de1f..6531f22 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryWriterContext.java
@@ -18,6 +18,9 @@
  */
 package org.apache.lens.cube.parse;
 
+import java.util.Map;
+
+import org.apache.lens.cube.metadata.Dimension;
 import org.apache.lens.server.api.error.LensException;
 
 /**
@@ -31,4 +34,5 @@ public interface QueryWriterContext {
   QueryAST getQueryAst();
   void updateFromString() throws LensException;
   QueryWriter toQueryWriter() throws LensException;
+  Map getDimsToQuery();
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/933cc3d6/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java 
b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
index caf8770..a27aec1 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -45,10 +45,10 @@ public final class RewriterPlan extends DriverQueryPlan {
 
   @SuppressWarnings("unchecked") // required for (Set) casting
   void extractPlan(Collection cubeQueries) {
-
 for (CubeQueryContext ctx : cubeQueries) {
-  if (ctx.getPickedDimTables() != null && 
!ctx.getPickedDimTables().isEmpty()) {
-for (CandidateDim dim : ctx.getPickedDimTables()) {
+  if (ctx.getQueryWriterContext().getDimsToQuery() != null
+  && !ctx.getQueryWriterContext().getDimsToQuery().isEmpty()) {
+for (CandidateDim dim : 
ctx.getQueryWriterContext().getDimsToQuery().values()) {
   addTablesQueried(dim.getStorageTable());
   if (partitions.get(dim.getName()) == null || 

lens git commit: LENS-1435: Cube xml files are missing in lens-example

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 89b17e640 -> aaf455111


LENS-1435: Cube xml files are missing in lens-example


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/aaf45511
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/aaf45511
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/aaf45511

Branch: refs/heads/master
Commit: aaf455111953fd1d49218a8bdf4bcc9600b01899
Parents: 89b17e6
Author: Sushil Mohanty 
Authored: Tue Jun 6 17:45:22 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 17:45:22 2017 +0530

--
 lens-examples/src/main/resources/cube11.xml | 53 
 lens-examples/src/main/resources/cube22.xml | 53 
 lens-examples/src/main/resources/cube33.xml | 53 
 3 files changed, 159 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube11.xml
--
diff --git a/lens-examples/src/main/resources/cube11.xml 
b/lens-examples/src/main/resources/cube11.xml
new file mode 100644
index 000..2283939
--- /dev/null
+++ b/lens-examples/src/main/resources/cube11.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube22.xml
--
diff --git a/lens-examples/src/main/resources/cube22.xml 
b/lens-examples/src/main/resources/cube22.xml
new file mode 100644
index 000..59b1828
--- /dev/null
+++ b/lens-examples/src/main/resources/cube22.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/lens/blob/aaf45511/lens-examples/src/main/resources/cube33.xml
--
diff --git a/lens-examples/src/main/resources/cube33.xml 
b/lens-examples/src/main/resources/cube33.xml
new file mode 100644
index 000..47b9b17
--- /dev/null
+++ b/lens-examples/src/main/resources/cube33.xml
@@ -0,0 +1,53 @@
+
+
+http://www.w3.org/2001/XMLSchema-instance;
+ xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

lens git commit: LENS-1427: ExpressionResolver is removing expressions if they are not answerable by *any* candidates

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master d3e732c9f -> cb2529672


LENS-1427: ExpressionResolver is removing expressions if they are not 
answerable by *any* candidates


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cb252967
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cb252967
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cb252967

Branch: refs/heads/master
Commit: cb252967237a7e6c945a9031a3d417c18bc736cf
Parents: d3e732c
Author: Rajat Khandelwal 
Authored: Tue Jun 6 15:37:41 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 15:37:41 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java | 37 +---
 .../test/resources/schema/cubes/base/b1cube.xml |  2 +-
 .../test/resources/schema/cubes/base/b2cube.xml |  2 +-
 .../resources/schema/cubes/base/testcube.xml|  2 +-
 4 files changed, 19 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/cb252967/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 7cad400..f38aa54 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -518,15 +518,14 @@ class ExpressionResolver implements ContextRewriter {
 }
 // Remove expressions for which denormalized columns are no more 
reachable
 esc.getDeNormCtx().pruneReferences(cubeql);
-for (String table : 
esc.getDeNormCtx().getTableToRefCols().keySet()) {
-  Set nonReachableFields = 
esc.getDeNormCtx().getNonReachableReferenceFields(table);
-  if (!nonReachableFields.isEmpty()) {
-log.info("Removing expression {} as columns {} are not 
available", esc, nonReachableFields);
-iterator.remove();
-removedEsc.add(esc);
-removed = true;
-break;
-  }
+if (!esc.getDeNormCtx().getTableToRefCols().isEmpty()
+  && esc.getDeNormCtx().getTableToRefCols().keySet().stream()
+  
.map(esc.getDeNormCtx()::getNonReachableReferenceFields).noneMatch(Set::isEmpty))
 {
+  log.info("Removing expression {} as all tables have non 
reachable fields", esc);
+  iterator.remove();
+  removedEsc.add(esc);
+  removed = true;
+  break;
 }
 if (removed) {
   continue;
@@ -630,19 +629,15 @@ class ExpressionResolver implements ContextRewriter {
 for (Map.Entry ecEntry : 
exprCtx.allExprsQueried.entrySet()) {
   String expr = ecEntry.getKey();
   Set ecSet = ecEntry.getValue();
-  for (ExpressionContext ec : ecSet) {
-if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) 
{
-  for (Iterator sItr = 
cubeql.getCandidates().iterator(); sItr.hasNext();) {
-Candidate cand = sItr.next();
-if (!cand.isExpressionEvaluable(ec)) {
-  log.info("Not considering Candidate :{} as {} is not 
evaluable", cand, ec.exprCol.getName());
-  sItr.remove();
-  cubeql.addCandidatePruningMsg(cand,
-  
CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
-}
-  }
+  cubeql.getCandidates().removeIf(x-> {
+if (ecSet.stream().noneMatch(x::isExpressionEvaluable)) {
+  log.info("Not considering Candidate :{} as {} is not evaluable", 
x, expr);
+  cubeql.addCandidatePruningMsg(x,
+CandidateTablePruneCause.expressionNotEvaluable(expr));
+  return true;
 }
-  }
+return false;
+  });
 }
   }
   // prune candidate dims without any valid expressions

http://git-wip-us.apache.org/repos/asf/lens/blob/cb252967/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
--
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml 
b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
index e3e9909..fd25982 100644
--- a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
+++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
@@ -178,7 +178,7 @@
   
 
 
-  
+  
 
 
   

lens git commit: LENS-1429: Queries failing intermittently with NO_REF_COL_AVAILABLE

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 159cf71e7 -> d3e732c9f


LENS-1429: Queries failing intermittently with NO_REF_COL_AVAILABLE


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d3e732c9
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d3e732c9
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d3e732c9

Branch: refs/heads/master
Commit: d3e732c9fe579b54d6c60e01a36746ecbd4ecb59
Parents: 159cf71
Author: Sushil Mohanty 
Authored: Tue Jun 6 15:35:46 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Jun 6 15:35:46 2017 +0530

--
 .../org/apache/lens/cube/parse/CandidateTableResolver.java | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d3e732c9/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 93736cd..f530650 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -674,11 +674,13 @@ class CandidateTableResolver implements ContextRewriter {
 if (colSet == null || colSet.isEmpty()) {
   return true;
 }
+boolean isEvaluable = false;
 for (QueriedPhraseContext qur : colSet) {
   if (sc.isPhraseAnswerable(qur)) {
-return true;
+isEvaluable = true;
+continue;
   }
 }
-return false;
+return isEvaluable;
   }
 }

lens git commit: LENS-1420 : Bug fixes in cube segmentation final rewritten query

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 139f029ae -> 7c1171c8d


LENS-1420 : Bug fixes in cube segmentation final rewritten query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7c1171c8
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7c1171c8
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7c1171c8

Branch: refs/heads/master
Commit: 7c1171c8dae842233d0db85bc15c00fcd37aa3de
Parents: 139f029
Author: Sushil Mohanty 
Authored: Mon May 15 13:27:54 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon May 15 13:27:54 2017 +0530

--
 .../lens/cube/parse/ExpressionResolver.java |  4 +++-
 .../lens/cube/parse/UnionQueryWriter.java   | 25 
 .../parse/TestCubeSegmentationRewriter.java | 22 -
 .../cube/parse/TestUnionAndJoinCandidates.java  | 13 ++
 .../resources/schema/cubes/base/b1c1cube.xml|  3 +++
 .../test/resources/schema/cubes/base/b1cube.xml |  3 +++
 .../resources/schema/cubes/base/b2c1cube.xml|  3 +++
 .../test/resources/schema/cubes/base/b2cube.xml |  3 +++
 .../resources/schema/cubes/base/basecube.xml|  7 ++
 .../resources/schema/cubes/base/testcube.xml|  3 +++
 .../cubes/derived/union_join_ctx_der1.xml   |  1 +
 11 files changed, 80 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index f86a84a..7cad400 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -426,14 +426,16 @@ class ExpressionResolver implements ContextRewriter {
   // Having ast is not copied now, it's maintained in cubeQueryContext, 
each fact processes that serially.
   if (queryAST.getHavingAST() != null) {
 replaceAST(cubeql, queryAST.getHavingAST());
-  } else {
+  } else if (cubeql.getHavingAST() != null) {
 replaceAST(cubeql, cubeql.getHavingAST());
+queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST()));
   }
   replaceAST(cubeql, queryAST.getOrderByAST());
 }
 
 private void replaceAST(final CubeQueryContext cubeql, ASTNode node) 
throws LensException {
   if (node == null) {
+
 return;
   }
   // Traverse the tree and resolve expression columns

http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index bd7134b..6c0d91a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -93,6 +93,9 @@ public class UnionQueryWriter extends SimpleHQLContext {
 for (StorageCandidateHQLContext sc : storageCandidates) {
   storageCandidateToSelectAstMap.put(sc.getStorageCandidate().toString(),
   new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT")));
+  if (sc.getQueryAst().getHavingAST() != null) {
+cubeql.setHavingAST(sc.getQueryAst().getHavingAST());
+  }
   sc.getQueryAst().setHavingAST(null);
   sc.getQueryAst().setOrderByAST(null);
   sc.getQueryAst().setLimitValue(null);
@@ -196,7 +199,11 @@ public class UnionQueryWriter extends SimpleHQLContext {
   ASTNode outerOrderby = new ASTNode(child);
   ASTNode tokNullsChild = (ASTNode) child.getChild(0);
   ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
-  outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new 
HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
+  if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == 
HiveParser.DOT) {
+outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new 
HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
+  } else {
+outerTokNullsChild.addChild(tokNullsChild);
+  }
   outerOrderby.addChild(outerTokNullsChild);
   outerExpression.addChild(outerOrderby);
 }
@@ -571,8 +578,12 @@ public class UnionQueryWriter extends SimpleHQLContext {
   return outerAST;
 } else {
   ASTNode outerAST = getDotAST(cubeql.getCube().getName(), alias);
-  (isSelectAst ? 

lens git commit: LENS-1418: LensClient is not thread safe

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 2d3a6cccd -> 139f029ae


LENS-1418: LensClient is not thread safe


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/139f029a
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/139f029a
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/139f029a

Branch: refs/heads/master
Commit: 139f029ae88da7cc29a02afeee73a542f0c92eef
Parents: 2d3a6cc
Author: Rajat Khandelwal 
Authored: Fri May 5 17:10:50 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri May 5 17:10:50 2017 +0530

--
 .../test/java/org/apache/lens/cli/TestLensQueryCommands.java   | 2 +-
 .../src/main/java/org/apache/lens/client/LensClient.java   | 6 +++---
 src/site/apt/user/cli.apt  | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/139f029a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
--
diff --git 
a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java 
b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index e1eaae5..2f80851 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -230,7 +230,7 @@ public class TestLensQueryCommands extends 
LensCliApplicationTest {
 //Fetch again. Should not get resultset
 result = qCom.getQueryResults(handle, null, true);
 log.debug("Prepared Query Result is  " + result);
-assertTrue(result.contains("Failed"), "Query is not purged yet " + handle);
+assertTrue(result.contains("not available"), "Query is not purged yet " + 
handle);
 
 result = qCom.destroyPreparedQuery(qh);
 

http://git-wip-us.apache.org/repos/asf/lens/blob/139f029a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
--
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java 
b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index e936798..4f8da06 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -226,10 +226,10 @@ public class LensClient implements AutoCloseable {
   throw new IllegalStateException(query.getStatus().getErrorMessage());
 }
 LensClientResultSet result = null;
-if (statement.getStatus().isResultSetAvailable()) {
-  result = new LensClientResultSet(statement.getResultSetMetaData(), 
statement.getResultSet());
+if (query.getStatus().isResultSetAvailable()) {
+  result = new LensClientResultSet(statement.getResultSetMetaData(query), 
statement.getResultSet(query));
 }
-return new LensClientResultSetWithStats(result, statement.getQuery());
+return new LensClientResultSetWithStats(result, query);
   }
 
   public LensClientResultSetWithStats getAsyncResults(QueryHandle q) {

http://git-wip-us.apache.org/repos/asf/lens/blob/139f029a/src/site/apt/user/cli.apt
--
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index caffda7..fce41fe 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -244,7 +244,7 @@ User CLI Commands
 *--+--+
 |fact get storage [--fact_name] \ [--storage_name] 
\|describe storage <<>> of fact 
<<>>\ |
 *--+--+
-|fact list partitions [--fact_name] \ [--storage_name] 
\ [[--filter] \]|get all partitions 
associated with fact <<>>, storage <<>> filtered by 
<<>>\ |
+|fact list partitions [--fact_name] \ [--storage_name] 
\ [--filter] \|get all partitions associated 
with fact <<>>, storage <<>> filtered by 
<<>>\ |
 *--+--+
 |fact list storage [--fact_name] \|display list of storages 
associated to fact <<>>\ |
 *--+--+

lens git commit: LENS-1417: Add filters in cli create schema command

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 2448336da -> 2d3a6cccd


LENS-1417: Add filters in cli create schema command


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2d3a6ccc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2d3a6ccc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2d3a6ccc

Branch: refs/heads/master
Commit: 2d3a6cccd259a081d489d4a8d4bb410229b67584
Parents: 2448336
Author: Rajat Khandelwal 
Authored: Fri May 5 12:20:04 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri May 5 12:20:04 2017 +0530

--
 .../lens/api/metastore/SchemaTraverser.java | 33 +-
 .../lens/cli/commands/LensSchemaCommands.java   | 68 
 .../apache/lens/cube/parse/CubeTestSetup.java   |  2 +-
 3 files changed, 60 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/2d3a6ccc/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
--
diff --git 
a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java 
b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
index 76cb8b9..9564443 100644
--- a/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
+++ b/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
@@ -19,19 +19,23 @@
 package org.apache.lens.api.metastore;
 
 import java.io.File;
-import java.io.FilenameFilter;
 import java.util.Map;
 import java.util.function.BiConsumer;
 
 import com.google.common.collect.Maps;
+import lombok.Data;
 
 /*
  * Created on 07/03/17.
  */
+@Data
 public class SchemaTraverser implements Runnable {
-  final File parent;
-  final Map types = Maps.newLinkedHashMap();
+  private final File parent;
   private final SchemaEntityProcessor action;
+  private final String entityTypeFilter;
+  private final String fileNameFilter;
+  final Map types = Maps.newLinkedHashMap();
+
   {
 types.put("storages", XStorage.class);
 types.put("cubes/base", XBaseCube.class);
@@ -43,26 +47,23 @@ public class SchemaTraverser implements Runnable {
 types.put("dimensiontables", XDimensionTable.class);
 types.put("segmentations", XSegmentation.class);
   }
-  private static final FilenameFilter XML_FILTER = (dir, name) -> 
name.endsWith(".xml");
 
   public interface SchemaEntityProcessor extends BiConsumer {
   }
 
-  public SchemaTraverser(File parent, SchemaEntityProcessor action) {
-this.parent = parent;
-this.action = action;
-  }
-
   @Override
   public void run() {
 for (Map.Entry entry : types.entrySet()) {
-  File f = new File(parent, entry.getKey());
-  if (f.exists()) {
-assert f.isDirectory();
-File[] files = f.listFiles(XML_FILTER);
-if (files != null) {
-  for (File entityFile : files) {
-action.accept(entityFile.getAbsoluteFile(), entry.getValue());
+  if (entityTypeFilter == null || 
entry.getKey().contains(entityTypeFilter)) {
+File f = new File(parent, entry.getKey());
+if (f.exists()) {
+  assert f.isDirectory();
+  File[] files = f.listFiles((dir, name) -> name.endsWith(".xml")
+&& (fileNameFilter == null || 
name.toLowerCase().contains(fileNameFilter.toLowerCase(;
+  if (files != null) {
+for (File entityFile : files) {
+  action.accept(entityFile.getAbsoluteFile(), entry.getValue());
+}
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/2d3a6ccc/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
--
diff --git 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
index 60bd9e0..20f313a 100644
--- 
a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
+++ 
b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java
@@ -19,7 +19,7 @@
 package org.apache.lens.cli.commands;
 
 import java.io.File;
-import java.io.FilenameFilter;
+import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -42,13 +42,14 @@ import org.springframework.shell.core.annotation.CliOption;
 import org.springframework.shell.support.logging.HandlerUtils;
 import org.springframework.stereotype.Component;
 
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 @Component
 @UserDocumentation(title = "Creating schema with one command",
   

lens git commit: LENS-1419: UnionCandidates getting pruned when only dim attributes projected

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master a899577ec -> 2448336da


LENS-1419: UnionCandidates getting pruned when only dim attributes projected


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/2448336d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/2448336d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/2448336d

Branch: refs/heads/master
Commit: 2448336da1033d0d037fcaa875650528612df161
Parents: a899577
Author: Sushil Mohanty 
Authored: Thu May 4 16:58:00 2017 +0530
Committer: Rajat Khandelwal 
Committed: Thu May 4 16:58:00 2017 +0530

--
 .../apache/lens/cube/parse/CandidateCoveringSetsResolver.java   | 4 +++-
 .../main/java/org/apache/lens/cube/parse/UnionQueryWriter.java  | 5 ++---
 2 files changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/2448336d/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index d6f8ad1..1e9873f 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -141,7 +141,9 @@ public class CandidateCoveringSetsResolver implements 
ContextRewriter {
 // prune non covering sets
 pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql);
 // prune candidate set which doesn't contain any common measure i
-pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
+if (!queriedMsrs.isEmpty()) {
+  pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, 
queriedMsrs);
+}
 // prune redundant covering sets
 pruneRedundantUnionCoveringSets(unionCoveringSet);
 // pruing done in the previous steps, now create union candidates

http://git-wip-us.apache.org/repos/asf/lens/blob/2448336d/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 01b39e9..bd7134b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -336,12 +336,11 @@ public class UnionQueryWriter extends SimpleHQLContext {
   if (phraseCountMap.get(col).size() > 1) {
 List childenToDelete = phraseCountMap.get(col).
 subList(1, phraseCountMap.get(col).size());
-int counter = 0;
 for (int i : childenToDelete) {
   for (StorageCandidateHQLContext sc : storageCandidates) {
-sc.getQueryAst().getSelectAST().deleteChild(i - counter);
+sc.getQueryAst().getSelectAST().setChild(i,
+new ASTNode(new CommonToken(HiveParser.Identifier, 
DUPLICATE_EXPRESSION_PREFIX)));
   }
-  counter++;
 }
   }
 }

lens git commit: LENS-1416 : Union query order by should work on column alias

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master b58749e20 -> a899577ec


LENS-1416 : Union query order by should work on column alias


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a899577e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a899577e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a899577e

Branch: refs/heads/master
Commit: a899577ecaca09a3c237c072fd94550cc80711c6
Parents: b58749e
Author: Sushil Mohanty 
Authored: Wed May 3 16:23:33 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed May 3 16:23:33 2017 +0530

--
 lens-api/src/main/resources/lens-errors.conf|  5 
 .../lens/cube/error/LensCubeErrorCode.java  |  1 +
 .../apache/lens/cube/parse/CandidateUtil.java   | 27 
 .../cube/parse/StorageCandidateHQLContext.java  | 23 +
 .../lens/cube/parse/UnionQueryWriter.java   |  1 +
 .../lens/cube/parse/TestUnionQueries.java   | 21 ---
 6 files changed, 52 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a899577e/lens-api/src/main/resources/lens-errors.conf
--
diff --git a/lens-api/src/main/resources/lens-errors.conf 
b/lens-api/src/main/resources/lens-errors.conf
index e5536bb..43de1e9 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -345,6 +345,11 @@ lensCubeErrorsForQuery = [
 errorMsg = "%s does not have any facts that can cover the queried measure 
set : %s"
   }
 
+  {
+errorCode = 3036
+httpStatusCode = ${BAD_REQUEST}
+errorMsg = "Order by column alias : %s shouldn't contain white space "
+  }
 
 ]
 

http://git-wip-us.apache.org/repos/asf/lens/blob/a899577e/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java 
b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index babe3de..32b9db3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -39,6 +39,7 @@ public enum LensCubeErrorCode {
   INVALID_TIME_RANGE(3014, 0),
   FROM_AFTER_TO(3015, 0),
   JOIN_TARGET_NOT_CUBE_TABLE(3016, 0),
+  ORDERBY_ALIAS_CONTAINING_WHITESPACE(3036, 0),
   // Error codes different for drivers
   CANNOT_USE_TIMERANGE_WRITER(3017, 100),
   NO_DEFAULT_AGGREGATE(3018, 200),

http://git-wip-us.apache.org/repos/asf/lens/blob/a899577e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java
index 467ca0a..6ba46d6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java
@@ -23,6 +23,10 @@ import static 
org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
 import java.util.*;
 import java.util.stream.Collectors;
 
+import org.apache.lens.cube.error.LensCubeErrorCode;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.HiveParser;
 
@@ -160,4 +164,27 @@ public final class CandidateUtil {
   static Set getColumnsFromCandidates(Collection 
scSet) {
 return 
scSet.stream().map(Candidate::getColumns).flatMap(Collection::stream).collect(Collectors.toSet());
   }
+
+  static void updateOrderByWithFinalAlias(ASTNode orderby, ASTNode select) 
throws LensException{
+if (orderby == null) {
+  return;
+}
+for (Node orderbyNode : orderby.getChildren()) {
+  ASTNode orderBychild = (ASTNode) orderbyNode;
+  for (Node selectNode : select.getChildren()) {
+ASTNode selectChild = (ASTNode) selectNode;
+if (selectChild.getChildCount() == 2) {
+  if (HQLParser.getString((ASTNode) selectChild.getChild(0))
+  .equals(HQLParser.getString((ASTNode) 
orderBychild.getChild(0 {
+ASTNode alias = new ASTNode((ASTNode) selectChild.getChild(1));
+if (!alias.toString().matches("\\S+")) {
+  throw new 
LensException(LensCubeErrorCode.ORDERBY_ALIAS_CONTAINING_WHITESPACE.getLensErrorInfo(),
 alias);
+}
+orderBychild.replaceChildren(0, 0, alias);
+break;
+  }
+}
+  }
+}
+  }
 }

[1/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master d78766c66 -> b58749e20


http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
--
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml 
b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
new file mode 100644
index 000..f0a80be
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml
@@ -0,0 +1,909 @@
+
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+  
+
+
+
+
+  
+
+
+
+
+  
+
+
+  
+  
+
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+
+  APAC
+  EMEA
+  USA
+
+  
+
+
+
+
+  
+
+
+
+
+  
+
+
+
+  
+  
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+  
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  

[2/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
--
diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml 
b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
new file mode 100644
index 000..ef0e33d
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml
@@ -0,0 +1,909 @@
+
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+  
+
+
+
+
+  
+
+
+
+
+  
+
+
+  
+  
+
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+
+  APAC
+  EMEA
+  USA
+
+  
+
+
+
+
+  
+
+
+
+
+  
+
+
+
+  
+  
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+
+
+  
+
+
+  
+
+
+  
+
+
+  
+  
+  
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+
+  
+
+  
+  
+
+
+  
+  
+  

[5/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
new file mode 100644
index 000..a359d86
--- /dev/null
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toMap;
+
+import static org.apache.lens.cube.metadata.DateUtil.formatAbsDate;
+import static org.apache.lens.cube.metadata.MetastoreUtil.getStringLiteralAST;
+
+import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_HAVING;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_INSERT;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_ORDERBY;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_SELEXPR;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.lens.cube.metadata.Cube;
+import org.apache.lens.cube.metadata.CubeColumn;
+import org.apache.lens.cube.metadata.CubeInterface;
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.Segment;
+import org.apache.lens.cube.metadata.Segmentation;
+import org.apache.lens.cube.metadata.TimeRange;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+
+import org.antlr.runtime.CommonToken;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import lombok.Getter;
+
+/**
+ * Created on 09/03/17.
+ */
+public class SegmentationCandidate implements Candidate {
+
+  Collection columns;
+  @Getter
+  private final CubeQueryContext cubeQueryContext;
+  private Segmentation segmentation;
+  private Map cubesOfSegmentation;
+  Map cubeQueryContextMap;
+  @Getter
+  private final Set answerableMeasurePhraseIndices = 
Sets.newHashSet();
+  private Map queriedRangeToMyRange = Maps.newHashMap();
+
+  SegmentationCandidate(CubeQueryContext cubeQueryContext, Segmentation 
segmentation) throws LensException {
+this.cubeQueryContext = cubeQueryContext;
+this.segmentation = segmentation;
+cubesOfSegmentation = Maps.newHashMap();
+cubeQueryContextMap = Maps.newHashMap();
+for (Segment segment : segmentation.getSegments()) {
+  // assuming only base cubes in segmentation
+  cubesOfSegmentation.put(segment.getName(), (Cube) 
getCubeMetastoreClient().getCube(segment.getName()));
+}
+  }
+
+
+  public SegmentationCandidate explode() throws LensException {
+return this;
+  }
+
+  private static  Predicate not(Predicate predicate) {
+return predicate.negate();
+  }
+
+  boolean rewriteInternal(Configuration conf, HiveConf hconf) throws 
LensException {
+CubeInterface cube = getCube();
+if (cube == null) {
+  return false;
+}
+for (Segment segment : segmentation.getSegments()) {
+  // assuming only base cubes in segmentation
+  Cube innerCube = (Cube) 
getCubeMetastoreClient().getCube(segment.getName());
+  cubesOfSegmentation.put(segment.getName(), innerCube);
+  Set notAnswerable = 
cubeQueryContext.getQueriedPhrases().stream()
+.filter(not(this::isPhraseAnswerable)).collect(Collectors.toSet());
+ 

[4/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
index 7f07dbc..757a877 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionCandidate.java
@@ -18,13 +18,27 @@
  */
 package org.apache.lens.cube.parse;
 
-import java.util.*;
+import static java.util.Comparator.comparing;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 
 import org.apache.lens.cube.metadata.FactPartition;
 import org.apache.lens.cube.metadata.TimeRange;
 import org.apache.lens.server.api.error.LensException;
 
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import lombok.Getter;
 
 /**
  * Represents a union of two candidates
@@ -34,20 +48,21 @@ public class UnionCandidate implements Candidate {
   /**
* Caching start and end time calculated for this candidate as it may have 
many child candidates.
*/
-  Date startTime = null;
-  Date endTime = null;
-  String toStr;
-  CubeQueryContext cubeql;
+  private Date startTime = null;
+  private Date endTime = null;
+  private String toStr;
+  @Getter
+  CubeQueryContext cubeQueryContext;
   /**
* List of child candidates that will be union-ed
*/
-  private List childCandidates;
-  private QueryAST queryAst;
+  @Getter
+  private List children;
+
   private Map> splitTimeRangeMap = 
Maps.newHashMap();
-  public UnionCandidate(List childCandidates, CubeQueryContext 
cubeql) {
-this.childCandidates = childCandidates;
-//this.alias = alias;
-this.cubeql = cubeql;
+  UnionCandidate(Collection childCandidates, 
CubeQueryContext cubeQueryContext) {
+this.children = Lists.newArrayList(childCandidates);
+this.cubeQueryContext = cubeQueryContext;
   }
 
   @Override
@@ -57,6 +72,15 @@ public class UnionCandidate implements Candidate {
   }
 
   @Override
+  public boolean isPhraseAnswerable(QueriedPhraseContext phrase) throws 
LensException {
+for (Candidate cand : getChildren()) {
+  if (!cand.isPhraseAnswerable(phrase)) {
+return false;
+  }
+}
+return true;
+  }
+
   public boolean isTimeRangeCoverable(TimeRange timeRange) throws 
LensException {
 Map candidateRange = getTimeRangeSplit(timeRange);
 for (Map.Entry entry : candidateRange.entrySet()) {
@@ -68,25 +92,52 @@ public class UnionCandidate implements Candidate {
   }
 
   @Override
+  public void addAnswerableMeasurePhraseIndices(int index) {
+for (Candidate candidate : getChildren()) {
+  candidate.addAnswerableMeasurePhraseIndices(index);
+}
+  }
+
+  @Override
+  public boolean isColumnValidForRange(String column) {
+return getChildren().stream().anyMatch(candidate -> 
candidate.isColumnValidForRange(column));
+  }
+
+  @Override
+  public Optional getColumnStartTime(String column) {
+return getChildren().stream()
+  .map(x->x.getColumnStartTime(column))
+  .filter(Optional::isPresent)
+  .map(Optional::get)
+  .min(Comparator.naturalOrder());
+  }
+
+  @Override
+  public Optional getColumnEndTime(String column) {
+return getChildren().stream()
+  .map(x->x.getColumnEndTime(column))
+  .filter(Optional::isPresent)
+  .map(Optional::get)
+  .max(Comparator.naturalOrder());
+  }
+
+
+  @Override
   public Collection getColumns() {
 // In UnionCandidate all columns are same, return the columns
 // of first child
-return childCandidates.iterator().next().getColumns();
+return children.iterator().next().getColumns();
   }
 
   @Override
   public Date getStartTime() {
 //Note: concurrent calls not handled specifically (This should not be a 
problem even if we do
 //get concurrent calls).
-
 if (startTime == null) {
-  Date minStartTime = childCandidates.get(0).getStartTime();
-  for (Candidate child : childCandidates) {
-if (child.getStartTime().before(minStartTime)) {
-  minStartTime = child.getStartTime();
-}
-  }
-  startTime = minStartTime;
+  startTime = children.stream()
+.map(Candidate::getStartTime)
+.min(Comparator.naturalOrder())
+.orElseGet(() -> new Date(Long.MIN_VALUE)); // this should be 
redundant.
 }
 return startTime;
   }
@@ -94,13 +145,10 @@ public class UnionCandidate implements Candidate {
   @Override
   public Date getEndTime() {
 if 

[3/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
index 3165d3a..57d28f3 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
@@ -19,9 +19,17 @@
 
 package org.apache.lens.cube.parse;
 
+import static org.apache.lens.cube.parse.HQLParser.getString;
+import static org.apache.lens.cube.parse.HQLParser.parseExpr;
+import static org.apache.lens.cube.parse.HQLParser.trimHavingAst;
+import static org.apache.lens.cube.parse.HQLParser.trimOrderByAst;
+
 import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 
+import static com.google.common.collect.Lists.newArrayList;
+
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -334,19 +342,19 @@ public class TestHQLParser {
 
   @Test
   public void testEqualsAST() throws Exception {
-ASTNode expr1 = HQLParser.parseExpr("T1.a + T2.b - T2.c");
-ASTNode expr2 = HQLParser.parseExpr("t1.A + t2.B - t2.C");
+ASTNode expr1 = parseExpr("T1.a + T2.b - T2.c");
+ASTNode expr2 = parseExpr("t1.A + t2.B - t2.C");
 
 Assert.assertTrue(HQLParser.equalsAST(expr1, expr2));
 
-ASTNode literalExpr1 = HQLParser.parseExpr("A = 'FooBar'");
-ASTNode literalExpr2 = HQLParser.parseExpr("a = 'FooBar'");
+ASTNode literalExpr1 = parseExpr("A = 'FooBar'");
+ASTNode literalExpr2 = parseExpr("a = 'FooBar'");
 Assert.assertTrue(HQLParser.equalsAST(literalExpr1, literalExpr2));
 
-ASTNode literalExpr3 = HQLParser.parseExpr("A = 'fOObAR'");
+ASTNode literalExpr3 = parseExpr("A = 'fOObAR'");
 Assert.assertFalse(HQLParser.equalsAST(literalExpr1, literalExpr3));
 
-ASTNode literalExpr4 = HQLParser.parseExpr("A <> 'FooBar'");
+ASTNode literalExpr4 = parseExpr("A <> 'FooBar'");
 Assert.assertFalse(HQLParser.equalsAST(literalExpr1, literalExpr4));
   }
 
@@ -393,7 +401,7 @@ public class TestHQLParser {
 
   @Test(dataProvider = "nAryFlatteningDataProvider")
   public void testNAryOperatorFlattening(String input, String expected) throws 
LensException {
-ASTNode tree = HQLParser.parseExpr(input);
+ASTNode tree = parseExpr(input);
 String infixString = HQLParser.getString(tree);
 Assert.assertEquals(infixString, expected);
   }
@@ -415,7 +423,7 @@ public class TestHQLParser {
   @Test(dataProvider = "colsInExpr")
   public void testColsInExpr(String input, String[] expected) throws Exception 
{
 String tableAlias = "cie";
-ASTNode inputAST = HQLParser.parseExpr(input);
+ASTNode inputAST = parseExpr(input);
 Set actual = HQLParser.getColsInExpr(tableAlias, inputAST);
 Set expectedSet = new HashSet<>(Arrays.asList(expected));
 Assert.assertEquals(actual, expectedSet, "Received " + actual + " for 
input:" + input);
@@ -446,7 +454,7 @@ public class TestHQLParser {
 
   @Test(dataProvider = "primitiveBool")
   public void testIsPrimitiveBooleanExpr(String input, boolean expected) 
throws Exception {
-ASTNode inputAST = HQLParser.parseExpr(input);
+ASTNode inputAST = parseExpr(input);
 boolean actual = HQLParser.isPrimitiveBooleanExpression(inputAST);
 Assert.assertEquals(actual, expected, "Received " + actual + " for input:" 
+ input + ":" + inputAST.dump());
   }
@@ -471,7 +479,7 @@ public class TestHQLParser {
 
   @Test(dataProvider = "primitiveBoolFunc")
   public void testIsPrimitiveBooleanFunction(String input, boolean expected) 
throws Exception {
-ASTNode inputAST = HQLParser.parseExpr(input);
+ASTNode inputAST = parseExpr(input);
 boolean actual = HQLParser.isPrimitiveBooleanFunction(inputAST);
 Assert.assertEquals(actual, expected, "Received " + actual + " for input:" 
+ input);
   }
@@ -503,7 +511,7 @@ public class TestHQLParser {
   @Test(dataProvider = "exprDataProvider")
   public void testParseExpr(String expr, HiveConf conf, boolean success) {
 try {
-  HQLParser.parseExpr(expr, conf);
+  parseExpr(expr, conf);
   Assert.assertTrue(success);
 } catch (LensException e) {
   Assert.assertFalse(success);
@@ -511,4 +519,40 @@ public class TestHQLParser {
   
Assert.assertTrue(e.getMessage().contains(LensCubeErrorCode.COULD_NOT_PARSE_EXPRESSION.name()));
 }
   }
+  @DataProvider
+  public Object[][] havingTrimDataProvider() {
+return new Object[][] {
+  {"((sum((testcube.segmsr1)) > 1) and (sum((testcube.msr2)) > 2))", 
newArrayList("segmsr1"),
+"(sum((testcube.segmsr1)) > 1)", },
+  {"(sum((testcube.msr2)) > 2)", newArrayList("segmsr1"), null, },
+  {"(sum((testcube.segmsr1)) > 1)", newArrayList("segmsr1"), 
"(sum((testcube.segmsr1)) > 1)", },
+

[7/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

LENS-974: Add cube-segmentation for base cube


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b58749e2
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b58749e2
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b58749e2

Branch: refs/heads/master
Commit: b58749e2061e0e731fc1855e0bf4a3b37c601c38
Parents: d78766c
Author: Rajat Khandelwal 
Authored: Tue May 2 17:25:57 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue May 2 17:25:58 2017 +0530

--
 lens-api/src/main/resources/cube-0.1.xsd|   4 +-
 .../lens/cli/commands/LensSchemaCommands.java   |  11 +-
 .../NoCandidateFactAvailableException.java  |   7 +-
 .../org/apache/lens/cube/metadata/Cube.java |  20 +-
 .../lens/cube/metadata/CubeFactTable.java   |  10 +
 .../org/apache/lens/cube/metadata/DateUtil.java |  15 +-
 .../lens/cube/metadata/MetastoreUtil.java   |  41 +-
 .../lens/cube/metadata/TimePartitionRange.java  |   6 +
 .../apache/lens/cube/metadata/TimeRange.java| 109 +--
 .../lens/cube/parse/AggregateResolver.java  |  13 +-
 .../apache/lens/cube/parse/AliasReplacer.java   |   3 +-
 .../org/apache/lens/cube/parse/Candidate.java   | 265 +-
 .../parse/CandidateCoveringSetsResolver.java| 135 +--
 .../lens/cube/parse/CandidateExploder.java  |  45 +
 .../cube/parse/CandidateTablePruneCause.java|  40 +
 .../lens/cube/parse/CandidateTableResolver.java | 105 +--
 .../apache/lens/cube/parse/CandidateUtil.java   | 216 +
 .../lens/cube/parse/CubeQueryConfUtil.java  |   1 +
 .../lens/cube/parse/CubeQueryContext.java   | 276 ++
 .../lens/cube/parse/CubeQueryRewriter.java  |  31 +-
 .../apache/lens/cube/parse/DefaultQueryAST.java |   8 +-
 .../cube/parse/DenormalizationResolver.java |  17 +-
 .../apache/lens/cube/parse/DimHQLContext.java   |  87 +-
 .../lens/cube/parse/DimOnlyHQLContext.java  |  61 +-
 .../lens/cube/parse/ExpressionResolver.java |  57 +-
 .../apache/lens/cube/parse/GroupbyResolver.java |   2 +-
 .../lens/cube/parse/HQLContextInterface.java|  85 --
 .../org/apache/lens/cube/parse/HQLParser.java   |  81 +-
 .../apache/lens/cube/parse/JoinCandidate.java   | 135 ++-
 .../apache/lens/cube/parse/JoinResolver.java|  30 +-
 .../lens/cube/parse/LeastPartitionResolver.java |   9 +-
 .../parse/MultiCandidateQueryWriterContext.java |  93 ++
 .../org/apache/lens/cube/parse/PruneCauses.java |  25 +-
 .../lens/cube/parse/QueriedPhraseContext.java   |  78 +-
 .../org/apache/lens/cube/parse/QueryAST.java|  24 +-
 .../org/apache/lens/cube/parse/QueryWriter.java |  31 +
 .../lens/cube/parse/QueryWriterContext.java |  34 +
 .../lens/cube/parse/SegmentationCandidate.java  | 382 
 .../cube/parse/SegmentationInnerRewriter.java   |  76 ++
 .../lens/cube/parse/SimpleHQLContext.java   |  85 +-
 .../lens/cube/parse/StorageCandidate.java   | 440 -
 .../cube/parse/StorageCandidateHQLContext.java  | 164 
 .../lens/cube/parse/StorageTableResolver.java   | 212 ++---
 .../org/apache/lens/cube/parse/StorageUtil.java |  21 +-
 .../lens/cube/parse/TimerangeResolver.java  |   2 +-
 .../apache/lens/cube/parse/UnionCandidate.java  | 163 ++--
 .../lens/cube/parse/UnionQueryWriter.java   | 173 ++--
 .../lens/cube/parse/join/AutoJoinContext.java   |  80 +-
 .../cube/parse/join/BridgeTableJoinContext.java |   8 +-
 .../apache/lens/cube/parse/join/JoinClause.java |  15 +-
 .../cube/metadata/TestCubeMetastoreClient.java  |   1 -
 .../apache/lens/cube/metadata/TestDateUtil.java |  14 +-
 .../apache/lens/cube/parse/CubeTestSetup.java   | 104 ++-
 .../lens/cube/parse/TestBaseCubeQueries.java|  36 +-
 .../lens/cube/parse/TestCubeRewriter.java   |  34 +-
 .../parse/TestCubeSegmentationRewriter.java | 322 +++
 .../cube/parse/TestDenormalizationResolver.java |   9 +-
 .../lens/cube/parse/TestExpressionResolver.java |   2 +-
 .../apache/lens/cube/parse/TestHQLParser.java   |  66 +-
 .../lens/cube/parse/TestJoinResolver.java   |  14 +-
 .../lens/cube/parse/TestQueryMetrics.java   |  38 +-
 .../lens/cube/parse/TestQueryRewrite.java   |  14 +-
 .../lens/cube/parse/TestTimeRangeResolver.java  |  11 +-
 .../lens/cube/parse/TestUnionQueries.java   |  13 +-
 .../resources/schema/cubes/base/b1c1cube.xml| 903 ++
 .../test/resources/schema/cubes/base/b1cube.xml | 909 +++
 .../resources/schema/cubes/base/b2c1cube.xml| 903 ++
 .../test/resources/schema/cubes/base/b2cube.xml | 909 +++
 .../resources/schema/cubes/base/basecube.xml|   4 +
 .../resources/schema/cubes/base/testcube.xml|  74 +-
 .../cubes/derived/union_join_ctx_der1.xml   |   1 +
 .../test/resources/schema/facts/b1b2fact1.xml   |  56 ++
 .../src/test/resources/schema/facts/b1fact1.xml |  85 ++
 

[6/7] lens git commit: LENS-974: Add cube-segmentation for base cube

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index c63c4c0..47bce65 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -29,8 +29,17 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
 import static com.google.common.base.Preconditions.checkArgument;
 
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.Predicate;
 
 import org.apache.lens.cube.error.LensCubeErrorCode;
@@ -66,6 +75,7 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST,
   static final String TIME_RANGE_FUNC = "time_range_in";
   public static final String NOW = "now";
   static final String DEFAULT_TABLE = "_default_";
+  @Getter
   private final ASTNode ast;
   @Getter
   private final QB qb;
@@ -179,7 +189,7 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST,
   @Setter
   private DenormalizationResolver.DenormalizationContext deNormCtx;
   @Getter
-  private PruneCauses  storagePruningMsgs = new 
PruneCauses<>();
+  private PruneCauses  storagePruningMsgs = new PruneCauses<>();
   @Getter
   private Map dimPruningMsgs =
 new HashMap();
@@ -462,13 +472,17 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
   }
 
   void addCandidatePruningMsg(Candidate cand, CandidateTablePruneCause 
pruneCause) {
-Set scs = CandidateUtil.getStorageCandidates(cand);
-for (StorageCandidate sc : scs) {
-  addStoragePruningMsg(sc, pruneCause);
+if (cand instanceof SegmentationCandidate) {
+  addStoragePruningMsg(cand, pruneCause);
+} else {
+  Set scs = CandidateUtil.getStorageCandidates(cand);
+  for (StorageCandidate sc : scs) {
+addStoragePruningMsg(sc, pruneCause);
+  }
 }
   }
 
-  void addStoragePruningMsg(StorageCandidate sc, CandidateTablePruneCause... 
factPruningMsgs) {
+  void addStoragePruningMsg(Candidate sc, CandidateTablePruneCause... 
factPruningMsgs) {
 for (CandidateTablePruneCause factPruningMsg: factPruningMsgs) {
   log.info("Pruning Storage {} with cause: {}", sc, factPruningMsg);
   storagePruningMsgs.addPruningMsg(sc, factPruningMsg);
@@ -627,14 +641,6 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST,
 }
   }
 
-  private void updateFromString(StorageCandidate sc, Map dimsToQuery) throws LensException {
-fromString = "%s"; // storage string is updated later
-if (isAutoJoinResolved()) {
-  fromString =
-getAutoJoinCtx().getFromString(fromString, sc, dimsToQuery.keySet(), 
dimsToQuery, this, this);
-}
-  }
-
   public String getSelectString() {
 return HQLParser.getString(selectAST);
   }
@@ -705,7 +711,7 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST,
   String getQBFromString(StorageCandidate candidate, Map dimsToQuery) throws LensException {
 String fromString;
 if (getJoinAST() == null) {
-  if (cube != null) {
+  if (candidate != null) {
 if (dimensions.size() > 0) {
   throw new 
LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
 }
@@ -772,25 +778,13 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST,
 
   void setNonexistingParts(Map nonExistingParts) throws 
LensException {
 if (!nonExistingParts.isEmpty()) {
-  ByteArrayOutputStream out = null;
-  String partsStr;
-  try {
-ObjectMapper mapper = new ObjectMapper();
-out = new ByteArrayOutputStream();
+  ObjectMapper mapper = new ObjectMapper();
+  try (ByteArrayOutputStream out = new ByteArrayOutputStream();){
 mapper.writeValue(out, nonExistingParts);
-partsStr = out.toString("UTF-8");
+conf.set(NON_EXISTING_PARTITIONS, out.toString("UTF-8"));
   } catch (Exception e) {
 throw new LensException("Error writing non existing parts", e);
-  } finally {
-if (out != null) {
-  try {
-out.close();
-  } catch (IOException e) {
-throw new LensException(e);
-  }
-}
   }
-  

lens git commit: LENS-1415: Make ResultRow implement Serializable

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 44a266a0f -> d78766c66


LENS-1415: Make ResultRow implement Serializable


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d78766c6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d78766c6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d78766c6

Branch: refs/heads/master
Commit: d78766c667dc9438ea1cd79aaf1c746c05e0f8ca
Parents: 44a266a
Author: Rajat Khandelwal 
Authored: Tue May 2 13:47:16 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue May 2 13:47:16 2017 +0530

--
 .../main/java/org/apache/lens/api/query/QueryResult.java| 6 --
 .../org/apache/lens/api/query/QueryResultSetMetadata.java   | 3 ++-
 .../main/java/org/apache/lens/api/query/ResultColumn.java   | 4 +++-
 .../java/org/apache/lens/api/query/ResultColumnType.java| 4 +++-
 .../src/main/java/org/apache/lens/api/query/ResultRow.java  | 3 ++-
 .../src/main/java/org/apache/lens/regression/util/Util.java | 9 -
 6 files changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d78766c6/lens-api/src/main/java/org/apache/lens/api/query/QueryResult.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryResult.java 
b/lens-api/src/main/java/org/apache/lens/api/query/QueryResult.java
index db91e41..0180617 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryResult.java
@@ -21,6 +21,8 @@
  */
 package org.apache.lens.api.query;
 
+import java.io.Serializable;
+
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 
@@ -35,9 +37,9 @@ import lombok.NoArgsConstructor;
  */
 @XmlRootElement
 @XmlSeeAlso({PersistentQueryResult.class, InMemoryQueryResult.class})
-/**
+/*
  * Instantiates a new query result.
  */
 @NoArgsConstructor(access = AccessLevel.PROTECTED)
-public abstract class QueryResult extends ToYAMLString implements 
PrettyPrintable {
+public abstract class QueryResult extends ToYAMLString implements 
PrettyPrintable, Serializable {
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/d78766c6/lens-api/src/main/java/org/apache/lens/api/query/QueryResultSetMetadata.java
--
diff --git 
a/lens-api/src/main/java/org/apache/lens/api/query/QueryResultSetMetadata.java 
b/lens-api/src/main/java/org/apache/lens/api/query/QueryResultSetMetadata.java
index dbdc677..67d9fb6 100644
--- 
a/lens-api/src/main/java/org/apache/lens/api/query/QueryResultSetMetadata.java
+++ 
b/lens-api/src/main/java/org/apache/lens/api/query/QueryResultSetMetadata.java
@@ -21,6 +21,7 @@
  */
 package org.apache.lens.api.query;
 
+import java.io.Serializable;
 import java.util.List;
 
 import javax.xml.bind.annotation.XmlElement;
@@ -49,7 +50,7 @@ import lombok.NoArgsConstructor;
  * Instantiates a new query result set metadata.
  */
 @NoArgsConstructor(access = AccessLevel.PROTECTED)
-public class QueryResultSetMetadata extends ToYAMLString {
+public class QueryResultSetMetadata extends ToYAMLString implements 
Serializable {
 
   /**
* The columns.

http://git-wip-us.apache.org/repos/asf/lens/blob/d78766c6/lens-api/src/main/java/org/apache/lens/api/query/ResultColumn.java
--
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/ResultColumn.java 
b/lens-api/src/main/java/org/apache/lens/api/query/ResultColumn.java
index ecf4713..b3c3869 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/ResultColumn.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/ResultColumn.java
@@ -21,6 +21,8 @@
  */
 package org.apache.lens.api.query;
 
+import java.io.Serializable;
+
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
@@ -46,7 +48,7 @@ import lombok.NoArgsConstructor;
  * Instantiates a new result column.
  */
 @NoArgsConstructor(access = AccessLevel.PROTECTED)
-public class ResultColumn {
+public class ResultColumn implements Serializable {
 
   /**
* The name.

http://git-wip-us.apache.org/repos/asf/lens/blob/d78766c6/lens-api/src/main/java/org/apache/lens/api/query/ResultColumnType.java
--
diff --git 
a/lens-api/src/main/java/org/apache/lens/api/query/ResultColumnType.java 
b/lens-api/src/main/java/org/apache/lens/api/query/ResultColumnType.java
index c71d02e..9df6597 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/ResultColumnType.java
+++ 

lens git commit: LENS-1414: StorageCandidates not getting pruned for non continuous update period

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master cb48aa386 -> 44a266a0f


LENS-1414: StorageCandidates not getting pruned for non continuous update period


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/44a266a0
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/44a266a0
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/44a266a0

Branch: refs/heads/master
Commit: 44a266a0fccf2162a66bf40e262e83c013ff776e
Parents: cb48aa3
Author: Sushil Mohanty 
Authored: Fri Apr 28 14:55:06 2017 +0530
Committer: Rajat Khandelwal 
Committed: Fri Apr 28 14:55:06 2017 +0530

--
 .../main/java/org/apache/lens/cube/parse/StorageTableResolver.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/44a266a0/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 10c3bbe..957b9ff 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -327,7 +327,7 @@ class StorageTableResolver implements ContextRewriter {
 //This is the prune cause
 pruningCauseForThisTimeRange =
   new 
CandidateTablePruneCause(CandidateTablePruneCode.TIME_RANGE_NOT_ANSWERABLE);
-  } else if (cubeql.shouldReplaceTimeDimWithPart()) {
+  } else if 
(!sc.getValidUpdatePeriods().contains(UpdatePeriod.CONTINUOUS)) {
 if (!client.partColExists(sc.getFact().getName(), 
sc.getStorageName(), range.getPartitionColumn())) {
   pruningCauseForThisTimeRange = 
partitionColumnsMissing(range.getPartitionColumn());
   TimeRange fallBackRange = StorageUtil.getFallbackRange(range, 
sc.getFact().getName(), cubeql);

lens git commit: LENS-1411: Queries not rewritten correctly when aggregate expression used inside "case when..."

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master de8f2ec6f -> 7f3731ef4


LENS-1411: Queries not rewritten correctly when aggregate expression used 
inside "case when..."


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7f3731ef
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7f3731ef
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7f3731ef

Branch: refs/heads/master
Commit: 7f3731ef49e7cc309620dd75bd679d7e8e29776e
Parents: de8f2ec
Author: Sushil Mohanty 
Authored: Mon Apr 24 14:56:29 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Apr 24 14:56:29 2017 +0530

--
 .../apache/lens/cube/parse/UnionQueryWriter.java | 19 +++
 1 file changed, 15 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/7f3731ef/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index 3ee817f..267d85b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -53,6 +53,7 @@ public class UnionQueryWriter {
   private CubeQueryContext cubeql;
   Collection storageCandidates;
   public static final String DEFAULT_MEASURE = "0.0";
+  public static final String DUPLICATE_EXPRESSION_PREFIX = "D";
 
   public UnionQueryWriter(Collection storageCandidates, 
CubeQueryContext cubeql) {
 if (storageCandidates == null || storageCandidates.size()<=1) {
@@ -353,15 +354,25 @@ public class UnionQueryWriter {
   if (phraseCountMap.get(col).size() > 1) {
 List childenToDelete = phraseCountMap.get(col).
 subList(1, phraseCountMap.get(col).size());
-int counter = 0;
 for (int i : childenToDelete) {
   for (StorageCandidate sc : storageCandidates) {
-sc.getQueryAst().getSelectAST().deleteChild(i - counter);
+sc.getQueryAst().getSelectAST().setChild(i,
+new ASTNode(new CommonToken(HiveParser.Identifier, 
DUPLICATE_EXPRESSION_PREFIX)));
   }
-  counter++;
 }
   }
 }
+
+for (StorageCandidate sc : storageCandidates) {
+  for (Node node : sc.getQueryAst().getSelectAST().getChildren()) {
+ASTNode selectNode = (ASTNode) node;
+if (selectNode.getToken().getType() == HiveParser.Identifier
+&& selectNode.getText().equals(DUPLICATE_EXPRESSION_PREFIX)) {
+  
sc.getQueryAst().getSelectAST().deleteChild(selectNode.getChildIndex());
+}
+  }
+}
+
 updateOuterASTDuplicateAliases(queryAst.getSelectAST(), aliasMap);
 if (queryAst.getHavingAST() != null) {
   updateOuterASTDuplicateAliases(queryAst.getHavingAST(), aliasMap);
@@ -719,7 +730,7 @@ public class UnionQueryWriter {
 ASTNode column = (ASTNode) selectExpr.getChild(0);
 if (HQLParser.isAggregateAST(column)
 && column.getChildCount() == 2) {
-  if (HQLParser.getString((ASTNode) column.getChild(1)).equals("0.0")) 
{
+  if (HQLParser.getString((ASTNode) 
column.getChild(1)).equals(DEFAULT_MEASURE)) {
 selectExpr.getParent().setChild(i, getSelectExpr(null, (ASTNode) 
selectExpr.getChild(1), true));
   }
 }

[08/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 7e06a5c..bab1080 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -35,8 +35,6 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.error.NoCandidateDimAvailableException;
 import org.apache.lens.cube.error.NoCandidateFactAvailableException;
 import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode;
 import org.apache.lens.server.api.LensServerAPITestUtil;
 import org.apache.lens.server.api.error.LensException;
 
@@ -53,7 +51,6 @@ import 
org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
-import com.google.common.base.Splitter;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import lombok.extern.slf4j.Slf4j;
@@ -81,7 +78,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
   @Test
   public void testQueryWithNow() throws Exception {
 LensException e = getLensExceptionInRewrite(
-  "select SUM(msr2) from testCube where " + getTimeRangeString("NOW - 
2DAYS", "NOW"), getConf());
+"select SUM(msr2) from testCube where " + getTimeRangeString("NOW - 
2DAYS", "NOW"), getConf());
 assertEquals(e.getErrorCode(), 
LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode());
   }
 
@@ -90,7 +87,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
 Configuration conf = getConf();
 conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true");
 conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, 
BetweenTimeRangeWriter.class, TimeRangeWriter.class);
-
+conf.setBoolean(REPLACE_TIMEDIM_WITH_PART_COL, false);
 DateFormat qFmt = new SimpleDateFormat("-MM-dd-HH:mm:ss");
 String timeRangeString;
 timeRangeString = getTimeRangeString(DAILY, -2, 0, qFmt);
@@ -99,15 +96,17 @@ public class TestCubeRewriter extends TestQueryRewrite {
 String to = getDateStringWithOffset(DAILY, 0, CONTINUOUS);
 String from = getDateStringWithOffset(DAILY, -2, CONTINUOUS);
 
-String expected = "select SUM((testCube.msr15)) from 
TestQueryRewrite.c0_testFact_CONTINUOUS testcube"
-  + " WHERE ((( testcube . dt ) between  '" + from + "'  and  '" + to + "' 
))";
+String expected = "select SUM((testCube.msr15))  as `sum(msr15)` from "
++ "TestQueryRewrite.c0_testFact_CONTINUOUS testcube"
++ " WHERE ((( testcube . d_time ) between  '" + from + "'  and  '" + 
to + "' ))";
 System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
 System.out.println("expected " + expected);
 compareQueries(rewrittenQuery.toHQL(), expected);
 
 //test with msr2 on different fact
+conf.setBoolean(REPLACE_TIMEDIM_WITH_PART_COL, true);
 rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where " + 
timeRangeString, conf);
-expected = "select SUM((testCube.msr2)) from TestQueryRewrite.c0_testFact 
testcube"
+expected = "select SUM((testCube.msr2)) as `sum(msr2)` from 
TestQueryRewrite.c2_testfact testcube"
   + " WHERE ((( testcube . dt ) between  '" + from + "'  and  '" + to + "' 
))";
 System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
 System.out.println("expected " + expected);
@@ -137,11 +136,12 @@ public class TestCubeRewriter extends TestQueryRewrite {
 CubeQueryContext rewrittenQuery =
   rewriteCtx("select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, 
getConfWithStorages("C2"));
 String expected =
-  getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) FROM ", 
null, null,
+  getExpectedQuery(TEST_CUBE_NAME, "select sum(testcube.msr2) as 
`sum(msr2)` FROM ", null, null,
 getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact"));
-compareQueries(rewrittenQuery.toHQL(), expected);
+String hql = rewrittenQuery.toHQL();
+compareQueries(hql, expected);
 System.out.println("Non existing parts:" + 
rewrittenQuery.getNonExistingParts());
-assertNotNull(rewrittenQuery.getNonExistingParts());
+//assertNotNull(rewrittenQuery.getNonExistingParts());
   }
 
   @Test
@@ -152,15 +152,9 @@ public class TestCubeRewriter extends TestQueryRewrite {
 conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2,C4");
 CubeQueryContext cubeQueryContext =
   rewriteCtx("select SUM(msr2) from testCube where " + THIS_YEAR_RANGE, 
conf);

[04/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/testdim3.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/testdim3.xml 
b/lens-cube/src/test/resources/schema/dimensions/testdim3.xml
new file mode 100644
index 000..6f7d295
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/testdim3.xml
@@ -0,0 +1,68 @@
+
+
+
+  
+
+
+
+
+
+
+  
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/testdim4.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/testdim4.xml 
b/lens-cube/src/test/resources/schema/dimensions/testdim4.xml
new file mode 100644
index 000..ca615bb
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/testdim4.xml
@@ -0,0 +1,45 @@
+
+
+
+  
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml 
b/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml
new file mode 100644
index 000..290448d
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/unreachabledim.xml
@@ -0,0 +1,44 @@
+
+
+
+  
+
+
+
+
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/user_interests.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/user_interests.xml 
b/lens-cube/src/test/resources/schema/dimensions/user_interests.xml
new file mode 100644
index 000..4a677a3
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/user_interests.xml
@@ -0,0 +1,50 @@
+
+
+
+  
+
+
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/userdim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/userdim.xml 
b/lens-cube/src/test/resources/schema/dimensions/userdim.xml
new file mode 100644
index 000..af5c720
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/userdim.xml
@@ -0,0 +1,78 @@
+
+
+
+  
+
+
+
+
+
+
+
+
+  
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimensions/zipdim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/zipdim.xml 
b/lens-cube/src/test/resources/schema/dimensions/zipdim.xml
new file mode 100644
index 000..29f3af6
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/zipdim.xml
@@ -0,0 +1,50 @@
+
+
+
+  
+
+
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/resources/schema/dimtables/citytable.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimtables/citytable.xml 
b/lens-cube/src/test/resources/schema/dimtables/citytable.xml
new file mode 100644
index 000..41ec206
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimtables/citytable.xml
@@ -0,0 +1,88 @@
+
+
+
+  
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+  
+  
+
+  
+HOURLY
+  
+  C1
+  
+
+  
+
+
+  
+
+
+  
+
+  
+
+
+  
+  C2
+  
+
+
+  
+  
+
+
+  
+
+  
+

[13/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
new file mode 100644
index 000..3ee817f
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -0,0 +1,730 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.cube.parse;
+
+import static org.apache.lens.cube.parse.HQLParser.*;
+
+import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
+
+import java.util.*;
+
+import org.apache.lens.cube.metadata.Dimension;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.util.StringUtils;
+
+import org.antlr.runtime.CommonToken;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Utility class to write union query. Given any complex Join or Union 
Candidate,
+ * this class rewrites union query for all the participating StorageCandidates.
+ */
+@Slf4j
+public class UnionQueryWriter {
+
+  private QueryAST queryAst;
+  private Map innerToOuterSelectASTs = new 
HashMap<>();
+  private Map innerToOuterHavingASTs = new 
HashMap<>();
+  private Map storageCandidateToSelectAstMap = new 
HashMap<>();
+  private AliasDecider aliasDecider = new DefaultAliasDecider();
+  private CubeQueryContext cubeql;
+  Collection storageCandidates;
+  public static final String DEFAULT_MEASURE = "0.0";
+
+  public UnionQueryWriter(Collection storageCandidates, 
CubeQueryContext cubeql) {
+if (storageCandidates == null || storageCandidates.size()<=1) {
+  throw new IllegalArgumentException("There should be atleast two storage 
candidates to write a union query");
+}
+this.cubeql = cubeql;
+this.storageCandidates = storageCandidates;
+  }
+
+  public String toHQL(Map factDimMap) throws 
LensException {
+StorageCandidate firstCandidate = storageCandidates.iterator().next();
+// Set the default queryAST for the outer query
+queryAst = DefaultQueryAST.fromStorageCandidate(firstCandidate,
+firstCandidate.getQueryAst());
+updateAsts();
+updateInnterSelectASTWithDefault();
+processSelectAndHavingAST();
+processGroupByAST();
+processOrderByAST();
+CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql);
+return CandidateUtil.buildHQLString(queryAst.getSelectString(), 
getFromString(factDimMap), null,
+queryAst.getGroupByString(), queryAst.getOrderByString(),
+queryAst.getHavingString(), queryAst.getLimitValue());
+  }
+
+  /**
+   * Set having, order by and limit clauses to null for inner queries
+   * being constructed from StorageCandidate.
+   */
+  private void updateAsts() {
+for (StorageCandidate sc : storageCandidates) {
+  storageCandidateToSelectAstMap.put(sc.toString(),
+  new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT")));
+  if (sc.getQueryAst().getHavingAST() != null) {
+sc.getQueryAst().setHavingAST(null);
+  }
+  if (sc.getQueryAst().getOrderByAST() != null) {
+sc.getQueryAst().setOrderByAST(null);
+  }
+  if (sc.getQueryAst().getLimitValue() != null) {
+sc.getQueryAst().setLimitValue(null);
+  }
+}
+  }
+
+  private void processGroupByAST() throws LensException {
+if (queryAst.getGroupByAST() != null) {
+  
queryAst.setGroupByAST(processGroupByExpression(queryAst.getGroupByAST()));
+}
+  }
+
+  /**
+   * Process havingAST for a StorageCandidate. Any column not projected and 
part of having clause
+   * project it in inner select
+   *
+   * @param innerAst
+   * @param aliasDecider
+   * @param sc
+   * @return ASTNode
+   * @throws LensException
+   */
+  private ASTNode 

[16/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index c99fdf1..926a4d0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,9 +25,6 @@ import java.util.*;
 
 import org.apache.lens.cube.metadata.*;
 import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
-import 
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
-import org.apache.lens.cube.parse.HQLParser.TreeNode;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -58,7 +55,7 @@ class ExpressionResolver implements ContextRewriter {
 private Map evaluableExpressions = 
new HashMap<>();
 private boolean hasMeasures = false;
 
-public boolean hasMeasures() {
+boolean hasMeasures() {
   return hasMeasures;
 }
 
@@ -148,11 +145,6 @@ class ExpressionResolver implements ContextRewriter {
 }
 
 void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, 
ExprSpecContext esc) throws LensException {
-  Set evalSet = evaluableExpressions.get(cTable);
-  if (evalSet == null) {
-evalSet = new LinkedHashSet<>();
-evaluableExpressions.put(cTable, evalSet);
-  }
   // add optional dimensions involved in expressions
   for (String table : esc.getTblAliasToColumns().keySet()) {
 if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && 
!srcAlias.equals(table)) {
@@ -161,7 +153,7 @@ class ExpressionResolver implements ContextRewriter {
   esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
 }
   }
-  evalSet.add(esc);
+  evaluableExpressions.computeIfAbsent(cTable, k -> new 
LinkedHashSet<>()).add(esc);
 }
 
 Set getAllASTNodes() {
@@ -182,13 +174,8 @@ class ExpressionResolver implements ContextRewriter {
 }
 
 boolean isEvaluable(CandidateTable cTable) {
-  if (directlyAvailableIn.contains(cTable)) {
-return true;
-  }
-  if (evaluableExpressions.get(cTable) == null) {
-return false;
-  }
-  return !evaluableExpressions.get(cTable).isEmpty();
+  return directlyAvailableIn.contains(cTable)
+|| (evaluableExpressions.get(cTable) != null && 
!evaluableExpressions.get(cTable).isEmpty());
 }
   }
 
@@ -252,16 +239,16 @@ class ExpressionResolver implements ContextRewriter {
   return null;
 }
 
-public boolean isValidInTimeRange(final TimeRange range) {
+boolean isValidInTimeRange(final TimeRange range) {
   return isValidFrom(range.getFromDate()) && 
isValidTill(range.getToDate());
 }
 
-public boolean isValidFrom(@NonNull final Date date) {
-  return (getStartTime() == null) ? true : date.equals(getStartTime()) || 
date.after(getStartTime());
+boolean isValidFrom(@NonNull final Date date) {
+  return (getStartTime() == null) || (date.equals(getStartTime()) || 
date.after(getStartTime()));
 }
 
-public boolean isValidTill(@NonNull final Date date) {
-  return (getEndTime() == null) ? true : date.equals(getEndTime()) || 
date.before(getEndTime());
+boolean isValidTill(@NonNull final Date date) {
+  return (getEndTime() == null) || (date.equals(getEndTime()) || 
date.before(getEndTime()));
 }
 
 public String toString() {
@@ -304,13 +291,7 @@ class ExpressionResolver implements ContextRewriter {
   this.cubeql = cubeql;
 }
 void addExpressionQueried(ExpressionContext expr) {
-  String exprCol = expr.getExprCol().getName().toLowerCase();
-  Set ecSet = allExprsQueried.get(exprCol);
-  if (ecSet == null) {
-ecSet = new LinkedHashSet();
-allExprsQueried.put(exprCol, ecSet);
-  }
-  ecSet.add(expr);
+  
allExprsQueried.computeIfAbsent(expr.getExprCol().getName().toLowerCase(), k -> 
new LinkedHashSet<>()).add(expr);
 }
 
 boolean isQueriedExpression(String column) {
@@ -337,7 +318,7 @@ class ExpressionResolver implements ContextRewriter {
   throw new IllegalArgumentException("no expression available for " + expr 
+ " alias:" + alias);
 }
 
-public boolean hasMeasures(String expr, CubeInterface cube) {
+boolean hasMeasures(String expr, CubeInterface cube) {
   String alias = cubeql.getAliasForTableName(cube.getName());
   

[09/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java
 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java
index cabb95e..5c57781 100644
--- 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java
+++ 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBetweenTimeRangeWriter.java
@@ -19,6 +19,8 @@
 
 package org.apache.lens.cube.parse;
 
+import static java.util.Optional.*;
+
 import static org.apache.lens.cube.metadata.DateFactory.*;
 import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY;
 
@@ -37,19 +39,19 @@ import org.testng.annotations.Test;
 
 public class TestBetweenTimeRangeWriter extends TestTimeRangeWriter {
 
-  public static final String CLOSED = "CLOSED";
-  public static final String OPEN = "OPEN";
+  private static final String CLOSED = "CLOSED";
+  private static final String OPEN = "OPEN";
 
-  public static final int START_DATE_OFFSET = 1;
-  public static final int END_DATE_OFFSET = 2;
+  private static final int START_DATE_OFFSET = 1;
+  private static final int END_DATE_OFFSET = 2;
 
-  public static final DateFormat DAY_DB_FORMAT = new 
SimpleDateFormat("-MM-dd");
+  private static final DateFormat DAY_DB_FORMAT = new 
SimpleDateFormat("-MM-dd");
 
   private static Set answeringParts;
   private static Set answeringPartsWithFormat;
 
   static {
-answeringParts = new LinkedHashSet();
+answeringParts = new LinkedHashSet<>();
 answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, 
START_DATE_OFFSET), DAILY, null, null));
 answeringParts.add(new FactPartition("dt", getDateWithOffset(DAILY, 
END_DATE_OFFSET), DAILY, null, null));
 
@@ -108,7 +110,7 @@ public class TestBetweenTimeRangeWriter extends 
TestTimeRangeWriter {
 
   }
 
-  public void validateBetweenOnlySingle(String whereClause, DateFormat format) 
{
+  private void validateBetweenOnlySingle(String whereClause, DateFormat 
format) {
 String expected = null;
 if (format == null) {
   expected =
@@ -121,22 +123,7 @@ public class TestBetweenTimeRangeWriter extends 
TestTimeRangeWriter {
 
   @DataProvider
   public Object[][] getBoundTypes() {
-
-Object[][] data = new Object[4][2];
-
-data[0][0] = OPEN;
-data[0][1] = OPEN;
-
-data[1][0] = OPEN;
-data[1][1] = CLOSED;
-
-data[2][0] = CLOSED;
-data[2][1] = OPEN;
-
-data[3][0] = CLOSED;
-data[3][1] = CLOSED;
-
-return data;
+return new Object[][]{{OPEN, OPEN}, {OPEN, CLOSED}, {CLOSED, OPEN}, 
{CLOSED, CLOSED}};
   }
 
   @Test(dataProvider = "getBoundTypes")
@@ -148,10 +135,10 @@ public class TestBetweenTimeRangeWriter extends 
TestTimeRangeWriter {
 
 int testStartOffset = START_DATE_OFFSET;
 int testEndOffset = END_DATE_OFFSET;
-if (startBoundType.equals(OPEN)) {
+if (startBoundType.equalsIgnoreCase(OPEN)) {
   testStartOffset = START_DATE_OFFSET - 1;
 }
-if (endBoundType.equals(OPEN)) {
+if (endBoundType.equalsIgnoreCase(OPEN)) {
   testEndOffset = END_DATE_OFFSET + 1;
 }
 validateBetweenBoundTypes(whereClause, null, testStartOffset, 
testEndOffset);
@@ -163,18 +150,11 @@ public class TestBetweenTimeRangeWriter extends 
TestTimeRangeWriter {
 validateBetweenBoundTypes(whereClause, DAY_DB_FORMAT, testStartOffset, 
testEndOffset);
   }
 
+
   private void validateBetweenBoundTypes(String whereClause, DateFormat format,
-int testStartOffset, int testEndOffset) {
-String expected = null;
-if (format == null) {
-  expected =
-getBetweenClause("test", "dt", getDateWithOffset(DAILY, 
testStartOffset),
-  getDateWithOffset(DAILY, testEndOffset), DAILY.format());
-} else {
-  expected =
-getBetweenClause("test", "dt", getDateWithOffset(DAILY, 
testStartOffset),
-  getDateWithOffset(DAILY, testEndOffset), format);
-}
+  int testStartOffset, int testEndOffset) {
+String expected = getBetweenClause("test", "dt", getDateWithOffset(DAILY, 
testStartOffset),
+  getDateWithOffset(DAILY, testEndOffset), 
ofNullable(format).orElseGet(DAILY::format));
 Assert.assertEquals(expected, whereClause);
   }
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
index 2f00244..9fbeb41 100644
--- 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBridgeTableQueries.java
+++ 

[14/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 2b63193..10c3bbe 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -7,7 +7,7 @@
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -18,35 +18,21 @@
  */
 package org.apache.lens.cube.parse;
 
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import static 
org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions;
+import static 
org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing;
 
 import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static org.apache.lens.cube.metadata.DateUtil.WSPACE;
-import static org.apache.lens.cube.metadata.MetastoreUtil.*;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
-import static 
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
-import static 
org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.*;
 
 import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.*;
+import 
org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
+import 
org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode;
 import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.metastore.*;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.util.ReflectionUtils;
 
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
 import lombok.extern.slf4j.Slf4j;
-
 /**
  * Resolve storages and partitions of all candidate tables and prunes 
candidate tables with missing storages or
  * partitions.
@@ -57,35 +43,13 @@ class StorageTableResolver implements ContextRewriter {
   private final Configuration conf;
   private final List supportedStorages;
   private final boolean allStoragesSupported;
-  CubeMetastoreClient client;
   private final boolean failOnPartialData;
   private final List validDimTables;
-  private final Map> 
validStorageMap = new HashMap<>();
-  private String processTimePartCol = null;
   private final UpdatePeriod maxInterval;
+  // TODO union : Remove this. All partitions are stored in the 
StorageCandidate.
   private final Map nonExistingPartitions = new 
HashMap<>();
-  private TimeRangeWriter rangeWriter;
-  private DateFormat partWhereClauseFormat = null;
+  private CubeMetastoreClient client;
   private PHASE phase;
-  private HashMap> 
skipStorageCausesPerFact;
-  private float completenessThreshold;
-  private String completenessPartCol;
-
-  enum PHASE {
-FACT_TABLES, FACT_PARTITIONS, DIM_TABLE_AND_PARTITIONS;
-
-static PHASE first() {
-  return values()[0];
-}
-
-static PHASE last() {
-  return values()[values().length - 1];
-}
-
-PHASE next() {
-  return values()[(this.ordinal() + 1) % values().length];
-}
-  }
 
   StorageTableResolver(Configuration conf) {
 this.conf = conf;
@@ -94,24 +58,13 @@ class StorageTableResolver implements ContextRewriter {
 this.failOnPartialData = 
conf.getBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
 String str = conf.get(CubeQueryConfUtil.VALID_STORAGE_DIM_TABLES);
 validDimTables = StringUtils.isBlank(str) ? null : 
Arrays.asList(StringUtils.split(str.toLowerCase(), ","));
-this.processTimePartCol = 
conf.get(CubeQueryConfUtil.PROCESS_TIME_PART_COL);
 String maxIntervalStr = conf.get(CubeQueryConfUtil.QUERY_MAX_INTERVAL);
 if (maxIntervalStr != null) {
-  this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr);
+  this.maxInterval = UpdatePeriod.valueOf(maxIntervalStr.toUpperCase());
 } else {
   this.maxInterval = null;
 }
-rangeWriter =
-  
ReflectionUtils.newInstance(conf.getClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS,
-CubeQueryConfUtil.DEFAULT_TIME_RANGE_WRITER, TimeRangeWriter.class), 
this.conf);
-String formatStr = 

[12/20] lens git commit: LENS-1381: Support Fact to Fact Union

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/ae83caae/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 9b29083..033264c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -27,29 +27,36 @@ import static org.apache.lens.cube.metadata.UpdatePeriod.*;
 
 import static org.testng.Assert.*;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
 import java.util.*;
+import java.util.stream.Collectors;
 
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.ToXMLString;
+import org.apache.lens.api.jaxb.LensJAXBContext;
+import org.apache.lens.api.metastore.SchemaTraverser;
 import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
-import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
 import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline;
 import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
 import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline;
-import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.error.LensException;
 
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.text.StrLookup;
+import org.apache.commons.lang3.text.StrSubstitutor;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.ParseException;
 import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.mapred.TextInputFormat;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -92,9 +99,6 @@ public class CubeTestSetup {
   public static final String TEST_CUBE_NAME = "testCube";
   public static final String DERIVED_CUBE_NAME = "derivedCube";
   public static final String BASE_CUBE_NAME = "baseCube";
-  public static final String DERIVED_CUBE_NAME1 = "der1";
-  public static final String DERIVED_CUBE_NAME2 = "der2";
-  public static final String DERIVED_CUBE_NAME3 = "der3";
 
   private static String c0 = "C0";
   private static String c1 = "C1";
@@ -137,7 +141,7 @@ public class CubeTestSetup {
 innerWhere, innerPostWhere, null, 
provider.providePartitionsForStorage(storage)));
   sep = " UNION ALL ";
 }
-return sb.append(") ").append(cubeName).append(" ").append(outerWhere == 
null ? "" : outerWhere)
+return sb.append(") ").append(" as ").append(cubeName).append(" 
").append(outerWhere == null ? "" : outerWhere)
   .append(" ").append(outerPostWhere == null ? "" : 
outerPostWhere).toString();
   }
   public static String getExpectedUnionQuery(String cubeName, List 
storages, StoragePartitionProvider provider,
@@ -329,9 +333,9 @@ public class CubeTestSetup {
 return updatePeriodToWhereMap;
   }
 
-  // storageTables[0] is hourly
-  // storageTables[1] is daily
-  // storageTables[2] is monthly
+  // storageName[0] is hourly
+  // storageName[1] is daily
+  // storageName[2] is monthly
   public static Map 
getWhereForMonthlyDailyAndHourly2months(String... storageTables) {
 Map storageTableToWhereClause = new LinkedHashMap();
 List hourlyparts = new ArrayList();
@@ -448,6 +452,16 @@ public class CubeTestSetup {
   StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
 return storageTableToWhereClause;
   }
+
+  public static Map getWhereForMonthly(String monthlyTable, 
Date startMonth, Date endMonth) {
+Map storageTableToWhereClause = new LinkedHashMap();
+List parts = new ArrayList();
+addParts(parts, MONTHLY, startMonth, endMonth);
+storageTableToWhereClause.put(getDbName() + monthlyTable,
+  StorageUtil.getWherePartClause("dt", TEST_CUBE_NAME, parts));
+return storageTableToWhereClause;
+  }
+
   public static Map getWhereForHourly2days(String hourlyTable) 
{
 return getWhereForHourly2days(TEST_CUBE_NAME, hourlyTable);
   }
@@ -511,1143 +525,14 @@ public class CubeTestSetup {
 return expected.toString();
   }
 
-  Set exprs;
-
-  private void createCube(CubeMetastoreClient client) 

[2/3] lens git commit: Merge with master

[prongs]

Merge with master


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3ba2fad1
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3ba2fad1
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3ba2fad1

Branch: refs/heads/lens-1381
Commit: 3ba2fad1091769a048a62d5b7bed3cebad8548e1
Parents: d45c538 186f03f
Author: Rajat Khandelwal 
Authored: Wed Apr 12 15:50:14 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Apr 12 15:50:14 2017 +0530

--
 .../main/resources/checkstyle-suppressions.xml  | 33 +++
 checkstyle/src/main/resources/checkstyle.xml| 12 --
 checkstyle/src/main/resources/suppressions.xml  | 32 ---
 contrib/clients/python/lens/client/main.py  |  7 +++-
 contrib/clients/python/lens/client/query.py | 43 +++-
 contrib/clients/python/setup.py |  2 +
 contrib/clients/python/test/test_lensclient.py  | 13 --
 .../java/org/apache/lens/api/ToXMLString.java   |  1 -
 .../lens/api/error/LensCommonErrorCode.java |  2 +-
 .../apache/lens/api/error/LensHttpStatus.java   |  2 +-
 .../lens/api/metastore/SchemaTraverser.java | 23 ---
 .../apache/lens/doc/TestGenerateConfigDoc.java  |  2 +-
 .../apache/lens/cube/metadata/CubeColumn.java   |  2 +-
 .../lens/cube/metadata/CubeMetastoreClient.java | 22 +-
 .../apache/lens/cube/metadata/Dimension.java|  4 +-
 .../lens/cube/metadata/MetastoreUtil.java   |  1 -
 .../org/apache/lens/cube/metadata/Storage.java  |  2 +
 .../apache/lens/cube/parse/AliasReplacer.java   |  4 --
 .../apache/lens/cube/parse/ColumnResolver.java  |  4 --
 .../lens/cube/parse/CubeQueryRewriter.java  | 17 
 .../cube/parse/DenormalizationResolver.java |  1 +
 .../apache/lens/cube/parse/GroupbyResolver.java | 11 ++---
 .../apache/lens/cube/parse/JoinResolver.java|  4 --
 .../lens/cube/parse/LeastPartitionResolver.java |  4 --
 .../cube/parse/LightestDimensionResolver.java   |  5 ---
 .../lens/cube/parse/LightestFactResolver.java   |  4 --
 .../cube/parse/MaxCoveringFactResolver.java |  2 +-
 .../lens/cube/parse/SelectPhraseContext.java|  2 +-
 .../lens/cube/parse/TimerangeResolver.java  |  3 --
 .../cube/metadata/TestCubeMetastoreClient.java  |  2 +-
 .../apache/lens/cube/parse/CubeTestSetup.java   |  1 -
 .../cube/parse/TestBetweenTimeRangeWriter.java  |  1 +
 .../lens/cube/parse/TestCubeRewriter.java   | 10 ++---
 .../lens/driver/es/ASTTraverserForES.java   |  6 +--
 .../client/jest/JestResultSetTransformer.java   |  4 +-
 .../lens/driver/es/translator/ESVisitor.java|  2 +-
 .../org/apache/lens/driver/hive/HiveDriver.java |  2 +-
 .../apache/lens/driver/jdbc/TestJdbcDriver.java |  2 +-
 .../core/helpers/SavedQueryResourceHelper.java  |  6 +--
 .../core/helpers/ScheduleResourceHelper.java|  8 ++--
 .../regression/client/ITSavedQueryTests.java|  2 +-
 .../server/api/events/SchedulerAlarmEvent.java  |  2 +-
 .../org/apache/lens/server/LensServices.java|  8 ++--
 .../metastore/CubeMetastoreServiceImpl.java |  1 -
 .../lens/server/scheduler/ScheduleResource.java | 18 
 .../server/user/UserConfigLoaderFactory.java|  8 ++--
 .../apache/lens/server/LensServerTestUtil.java  |  6 +--
 .../org/apache/lens/server/TestServerMode.java  | 24 +--
 .../lens/server/query/TestEventService.java |  4 +-
 .../TestQueryIndependenceFromSessionClose.java  |  6 ++-
 .../TotalQueryCostCeilingConstraintTest.java|  2 +-
 .../lens/server/query/retry/QueryRetryTest.java |  6 +--
 .../server/stats/TestLogStatisticsStore.java|  2 +-
 pom.xml | 21 +-
 tools/conf/server/lens-site.xml |  6 +++
 55 files changed, 236 insertions(+), 188 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/3ba2fad1/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
--
diff --cc lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
index 746a82b,e74adc9..0058f20
--- a/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
+++ b/lens-api/src/main/java/org/apache/lens/api/ToXMLString.java
@@@ -24,10 -24,6 +24,9 @@@ import java.util.HashMap
  import java.util.Map;
  
  import javax.xml.bind.*;
 +import javax.xml.bind.annotation.XmlRootElement;
- import javax.xml.bind.annotation.XmlSeeAlso;
 +import javax.xml.bind.annotation.XmlType;
 +import javax.xml.namespace.QName;
  
  import org.apache.lens.api.jaxb.LensJAXBContext;
  

http://git-wip-us.apache.org/repos/asf/lens/blob/3ba2fad1/lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java
--
diff --cc 

svn commit: r1790178 - /lens/doap.rdf

[prongs]

Author: prongs
Date: Wed Apr  5 06:12:26 2017
New Revision: 1790178

URL: http://svn.apache.org/viewvc?rev=1790178=rev
Log:
Corrected category and other information for apache lens

Modified:
lens/doap.rdf

Modified: lens/doap.rdf
URL: 
http://svn.apache.org/viewvc/lens/doap.rdf?rev=1790178=1790177=1790178=diff
==
--- lens/doap.rdf (original)
+++ lens/doap.rdf Wed Apr  5 06:12:26 2017
@@ -27,17 +27,17 @@
 Apache Lens
 http://lens.apache.org/; />
 http://lens.apache.org; />
-Unified Analytic Interface
+Unified Analytics Interface
 Lens provides an Unified Analytics interface.
   Lens aims to cut the Data Analytics silos by providing a single view of 
data across
   multiple tiered data stores and optimal execution environment for the 
analytical query.
   It seamlessly integrates Hadoop with traditional data warehouses to 
appear like one.
 
 https://issues.apache.org/jira/browse/LENS; />
-http://lens.apache.org/lenshome/quick-start.html; />
+http://lens.apache.org/mail-lists.html; />
 http://lens.apache.org/releases/download.html; />
 Java
-http://projects.apache.org/category/build-management; />
+http://projects.apache.org/category/big-data; />
 
   
 Apache Lens
@@ -91,13 +91,13 @@
   
 Apache Lens
 2016-10-28
-2.6.1-beta
+2.6.1
   
 
 
   
 https://git-wip-us.apache.org/repos/asf/lens.git"/>
-https://git-wip-us.apache.org/repos/asf?p=lens.git"/>
+https://github.com/apache/lens"/>
   
 
   

[05/16] lens git commit: LENS-1384 : Update error message for data completeness check

[prongs]

LENS-1384 : Update error message for data completeness check


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d46e78cf
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d46e78cf
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d46e78cf

Branch: refs/heads/lens-1381
Commit: d46e78cff644b8436b80adc6f441799cfda4af27
Parents: 29568fa
Author: Narayan Periwal 
Authored: Fri Jan 27 10:11:33 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Fri Jan 27 10:11:33 2017 +0530

--
 .../apache/lens/cube/parse/CandidateTablePruneCause.java| 9 +
 .../java/org/apache/lens/cube/parse/TestCubeRewriter.java   | 6 ++
 2 files changed, 7 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d46e78cf/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 2ad6e20..bd6e27c 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -148,15 +148,16 @@ public class CandidateTablePruneCause {
   }
 },
 // incomplete data in the fact
-INCOMPLETE_PARTITION("Data is incomplete. Details : %s") {
+INCOMPLETE_PARTITION("Data for the requested metrics is only partially 
complete. Partially complete metrics are:"
++ " %s. Please try again later or rerun after removing incomplete 
metrics") {
   Object[] getFormatPlaceholders(Set causes) {
-Set>> incompletePartitions = 
Sets.newHashSet();
+Set incompleteMetrics = Sets.newHashSet();
 for (CandidateTablePruneCause cause : causes) {
   if (cause.getIncompletePartitions() != null) {
-incompletePartitions.add(cause.getIncompletePartitions());
+incompleteMetrics.addAll(cause.getIncompletePartitions().keySet());
   }
 }
-return new String[]{incompletePartitions.toString()};
+return new String[]{incompleteMetrics.toString()};
   }
 };
 

http://git-wip-us.apache.org/repos/asf/lens/blob/d46e78cf/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index c9e7c29..98b021b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -970,11 +970,9 @@ public class TestCubeRewriter extends TestQueryRewrite {
 assertEquals(e.getErrorCode(), 
LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode());
 NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) 
e;
 PruneCauses.BriefAndDetailedError pruneCauses = ne.getJsonMessage();
-/*Since the Flag FAIL_QUERY_ON_PARTIAL_DATA is set, and thhe queried fact 
has incomplete data, hence, we expect the
+/*Since the Flag FAIL_QUERY_ON_PARTIAL_DATA is set, and the queried fact 
has incomplete data, hence, we expect the
 prune cause to be INCOMPLETE_PARTITION. The below check is to validate 
this.*/
-assertEquals(pruneCauses.getBrief().substring(0, 
INCOMPLETE_PARTITION.errorFormat.length() - 3),
-INCOMPLETE_PARTITION.errorFormat.substring(0,
-INCOMPLETE_PARTITION.errorFormat.length() - 3), 
pruneCauses.getBrief());
+assertEquals(pruneCauses.getBrief(), 
String.format(INCOMPLETE_PARTITION.errorFormat, "[msr9]"));
   }
 
   @Test

[01/16] lens git commit: LENS-1379 : Fix session expiry for sessions in which operations were done

[prongs]

Repository: lens
Updated Branches:
  refs/heads/lens-1381 d45c5384c -> 8868b0638


LENS-1379 : Fix session expiry for sessions in which operations were done


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/98990c39
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/98990c39
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/98990c39

Branch: refs/heads/lens-1381
Commit: 98990c39f4f4826beaf59afb0ef9961f566000c3
Parents: fe66131
Author: Amareshwari Sriramadasu 
Authored: Mon Jan 2 17:45:24 2017 +0530
Committer: Puneet 
Committed: Mon Jan 2 17:45:24 2017 +0530

--
 .../lens/server/session/HiveSessionService.java | 58 
 .../lens/server/session/LensSessionImpl.java| 20 +++---
 .../TestQueryIndependenceFromSessionClose.java  | 71 +---
 3 files changed, 102 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/98990c39/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
 
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 21e2a62..b480d14 100644
--- 
a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ 
b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -276,17 +276,19 @@ public class HiveSessionService extends BaseLensService 
implements SessionServic
*/
   @Override
   public void setSessionParameter(LensSessionHandle sessionid, String key, 
String value) {
-setSessionParameter(sessionid, key, value, true);
+HashMap config = Maps.newHashMap();
+config.put(key, value);
+setSessionParameters(sessionid, config);
   }
+
   /**
* Sets the session parameter.
*
* @param sessionidthe sessionid
* @param config   map of string-string. each entry represents key and 
the value to be set for that key
-   * @param addToSession the add to session
*/
 
-  protected void setSessionParameters(LensSessionHandle sessionid, Map config, boolean addToSession) {
+  protected void setSessionParameters(LensSessionHandle sessionid, Map config) {
 log.info("Request to Set params:" + config);
 try {
   acquire(sessionid);
@@ -297,17 +299,11 @@ public class HiveSessionService extends BaseLensService 
implements SessionServic
   var = var.substring(SystemVariables.HIVECONF_PREFIX.length());
 }
 getSession(sessionid).getSessionConf().set(var, entry.getValue());
-if (addToSession) {
-  String command = "set" + " " + entry.getKey() + "= " + 
entry.getValue();
-  
closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid),
 command, null));
-} else {
-  getSession(sessionid).getHiveConf().set(entry.getKey(), 
entry.getValue());
-}
+String command = "set" + " " + entry.getKey() + "= " + 
entry.getValue();
+
closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid),
 command, null));
   }
   // add to persist
-  if (addToSession) {
-getSession(sessionid).setConfig(config);
-  }
+  getSession(sessionid).setConfig(config);
   log.info("Set params:" + config);
 } catch (HiveSQLException e) {
   throw new WebApplicationException(e);
@@ -315,18 +311,18 @@ public class HiveSessionService extends BaseLensService 
implements SessionServic
   release(sessionid);
 }
   }
-/**
- * Sets the session parameter.
- *
- * @param sessionidthe sessionid
- * @param key  the key
- * @param valuethe value
- * @param addToSession the add to session
- */
-  protected void setSessionParameter(LensSessionHandle sessionid, String key, 
String value, boolean addToSession) {
-HashMap config = Maps.newHashMap();
-config.put(key, value);
-setSessionParameters(sessionid, config, addToSession);
+
+  private void setSessionParametersOnRestore(LensSessionHandle sessionid, 
Map config) {
+// set in session conf
+for(Map.Entry entry: config.entrySet()) {
+  String var = entry.getKey();
+  if (var.indexOf(SystemVariables.HIVECONF_PREFIX) == 0) {
+var = var.substring(SystemVariables.HIVECONF_PREFIX.length());
+  }
+  getSession(sessionid).getSessionConf().set(var, entry.getValue());
+  getSession(sessionid).getHiveConf().set(entry.getKey(), 
entry.getValue());
+}
+log.info("Set params on restart:" + config);
   }
 
   /*
@@ 

[10/16] lens git commit: LENS-1387: Move Lens compile & target version to Java 8

[prongs]

LENS-1387: Move Lens compile & target version to Java 8


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/0cd22b10
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/0cd22b10
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/0cd22b10

Branch: refs/heads/lens-1381
Commit: 0cd22b107ba311220a3038827f6977aa460cdcfe
Parents: 97fe26d
Author: Rajat Khandelwal 
Authored: Thu Feb 16 08:54:46 2017 +0800
Committer: raju 
Committed: Thu Feb 16 08:54:46 2017 +0800

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/0cd22b10/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 8ea64b7..12cd9cc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,8 +34,8 @@
 UTF-8
 
 
-1.7
-1.7
+1.8
+1.8
 3.0.1
 2.5
 2.0.1

[04/16] lens git commit: LENS-1373 : Specify chain of comparators for query comparator config

[prongs]

LENS-1373 : Specify chain of comparators for query comparator config


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/29568fa3
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/29568fa3
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/29568fa3

Branch: refs/heads/lens-1381
Commit: 29568fa3999642d017977b5dda3d7e08bee91337
Parents: 5ac6880
Author: Rajat Khandelwal 
Authored: Thu Jan 19 10:11:27 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Thu Jan 19 10:11:27 2017 +0530

--
 .../server/query/QueryExecutionServiceImpl.java |   3 +-
 .../src/main/resources/lensserver-default.xml   |  11 ++
 src/site/apt/admin/config.apt   | 128 ++-
 3 files changed, 77 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/29568fa3/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
 
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index c76ad24..c6fbeda 100644
--- 
a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ 
b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -404,8 +404,7 @@ public class QueryExecutionServiceImpl extends 
BaseLensService implements QueryE
   private void loadQueryComparator() throws LensException {
 try {
   Class[] classes = conf.getClasses(QUERY_COMPARATOR_CLASSES,
-MoreRetriesFirstComparator.class, QueryPriorityComparator.class,
-FIFOQueryComparator.class, QueryCostComparator.class);
+MoreRetriesFirstComparator.class, QueryPriorityComparator.class, 
FIFOQueryComparator.class);
   List comparators = Lists.newArrayList();
   for (Class clazz: classes) {
 comparators.add(clazz.asSubclass(QueryComparator.class).newInstance());

http://git-wip-us.apache.org/repos/asf/lens/blob/29568fa3/lens-server/src/main/resources/lensserver-default.xml
--
diff --git a/lens-server/src/main/resources/lensserver-default.xml 
b/lens-server/src/main/resources/lensserver-default.xml
index 261fa52..28b1db2 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -942,6 +942,17 @@
 
org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser
 The Query cost parser class. Default query cost class used is 
FactPartitionBasedQueryCost
   
+
+  
+lens.server.query.comparator.classes
+
org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator
+The Query cost comparator chain. Queries are compared in this 
order. To compare queries q1 and q2,
+first number of retries are considered. The one with more retries is 
placed first in the queue. If those are same,
+then their priorities are considered, with higher priorities coming before 
lower ones. If those are also same,
+then their submission times are considered. The query that was submitted 
first is placed first.
+
+  
+
   
 lens.cube.metastore.enable.datacompleteness.check
 false

http://git-wip-us.apache.org/repos/asf/lens/blob/29568fa3/src/site/apt/admin/config.apt
--
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index eb35ae3..8720c82 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -163,132 +163,134 @@ Lens server configuration
 *--+--+---+--+
 |68|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors 
are consulted first, before anything happens for the given query. They can 
either return null or return a messaging indicating why the given query 
shouldn't be accepted. These can be used to filter out queries at the earliest.|
 *--+--+---+--+
-|69|lens.server.query.cost.parser.class|org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser|The
 Query cost parser class. Default query cost class used is 
FactPartitionBasedQueryCost|
+|69|lens.server.query.comparator.classes|org.apache.lens.server.api.query.comparators.MoreRetriesFirstComparator,org.apache.lens.server.api.query.comparators.QueryPriorityComparator,org.apache.lens.server.api.query.comparators.FIFOQueryComparator|The
 Query cost comparator chain. Queries are compared in this order. To compare 
queries q1 

[07/16] lens git commit: LENS-1380 : Revamp testcase division

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/97fe26d3/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java
--
diff --git 
a/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java
 
b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java
index aeae8aa..1a7a391 100644
--- 
a/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java
+++ 
b/lens-regression/src/test/java/org/apache/lens/regression/throttling/ITCostTests.java
@@ -19,7 +19,6 @@
 
 package org.apache.lens.regression.throttling;
 
-import java.io.IOException;
 import java.lang.reflect.Method;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -37,7 +36,6 @@ import org.apache.lens.regression.core.helpers.*;
 import org.apache.lens.regression.core.testHelper.BaseTestClass;
 import org.apache.lens.regression.util.Util;
 import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.util.LensUtil;
 
 import org.apache.log4j.Logger;
@@ -45,9 +43,6 @@ import org.apache.log4j.Logger;
 import org.testng.Assert;
 import org.testng.annotations.*;
 
-import com.jcraft.jsch.JSchException;
-import com.jcraft.jsch.SftpException;
-
 public class ITCostTests extends BaseTestClass {
 
   private WebTarget servLens;
@@ -64,7 +59,7 @@ public class ITCostTests extends BaseTestClass {
   public static final String JDBC_QUERY1 = 
QueryInventory.getQueryFromInventory("JDBC.QUERY1");
 
   private static String hiveDriver = "hive/hive1";
-  private String hiveDriverSitePath  = lens.getServerDir() + 
"/conf/drivers/hive/hive1/hivedriver-site.xml";
+  private String lensSitePath  = lens.getServerDir() + "/conf/lens-site.xml";
   private static final long SECONDS_IN_A_MINUTE = 60;
 
   private static Logger logger = Logger.getLogger(ITCostTests.class);
@@ -75,6 +70,11 @@ public class ITCostTests extends BaseTestClass {
 logger.info("Creating a new Session");
 sessionHandleString = sHelper.openSession(lens.getCurrentDB());
 sHelper.setAndValidateParam(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, 
"false");
+
+HashMap map = 
LensUtil.getHashMap(LensConfConstants.TOTAL_QUERY_COST_CEILING_PER_USER_KEY, 
"60",
+LensConfConstants.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, 
DriverConfig.USER_COST_CONSTRAINT_FACTORY);
+Util.changeConfig(map, lensSitePath);
+lens.restart();
   }
 
   @BeforeMethod(alwaysRun = true)
@@ -94,29 +94,10 @@ public class ITCostTests extends BaseTestClass {
   public void closeSession() throws Exception {
 logger.info("Closing Session");
 sHelper.closeSession();
-  }
-
-  @BeforeGroups("user-cost-ceiling")
-  public void setUserCeilingconfig() throws Exception {
-try{
-  HashMap map = 
LensUtil.getHashMap(LensConfConstants.TOTAL_QUERY_COST_CEILING_PER_USER_KEY, 
"60",
-  DriverConfig.HIVE_CONSTRAINT_FACTORIES,
-  DriverConfig.MAX_CONCURRENT_CONSTRAINT_FACTORY + "," + 
DriverConfig.USER_COST_CONSTRAINT_FACTORY,
-  DriverConfig.MAX_CONCURRENT_QUERIES, "10");
-  Util.changeConfig(map, hiveDriverSitePath);
-  lens.restart();
-}catch (Exception e){
-  logger.info(e);
-}
-  }
-
-  @AfterGroups("user-cost-ceiling")
-  public void restoreConfig() throws SftpException, JSchException, 
InterruptedException, LensException, IOException {
-Util.changeConfig(hiveDriverSitePath);
+Util.changeConfig(lensSitePath);
 lens.restart();
   }
 
-
   @Test(enabled = true, groups= "user-cost-ceiling")
   public void testUserCostCeiling() throws Exception {
 
@@ -293,148 +274,5 @@ public class ITCostTests extends BaseTestClass {
 
   //TODO : Add queue level throttling along with user ceiling constraint
 
-  /*
-  * LENS-995 : Queue number shouldn't change with in the same prority
-  */
-
-  @Test(enabled = true)
-  public void queueNumberChangeWithInSamePriority() throws Exception {
-
-String longRunningQuery = 
String.format(QueryInventory.getQueryFromInventory("HIVE.SLEEP_COST_95"), "20");
-HashMap map = 
LensUtil.getHashMap(DriverConfig.MAX_CONCURRENT_QUERIES, "1");
-String[] queries = {longRunningQuery, COST_5, COST_5, COST_3, COST_2};
-
-try {
-  Util.changeConfig(map, hiveDriverSitePath);
-  lens.restart();
-
-  List handleList = new ArrayList<>();
-  for(String query : queries){
-handleList.add((QueryHandle) qHelper.executeQuery(query).getData());
-  }
-
-  LensQuery lq1 = qHelper.getLensQuery(sessionHandleString, 
handleList.get(1));
-  LensQuery lq2 = qHelper.getLensQuery(sessionHandleString, 
handleList.get(2));
-  LensQuery lq3 = qHelper.getLensQuery(sessionHandleString, 
handleList.get(3));
-  LensQuery lq4 = qHelper.getLensQuery(sessionHandleString, 
handleList.get(4));
-
-  

[14/16] lens git commit: LENS-1398: Python client not raising exception in some cases

[prongs]

LENS-1398: Python client not raising exception in some cases


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a3045fee
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a3045fee
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a3045fee

Branch: refs/heads/lens-1381
Commit: a3045fee12b3e750b6c931f5221a480d415efc76
Parents: bd344c7
Author: Rajat Khandelwal 
Authored: Tue Mar 21 16:09:05 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Mar 21 16:09:05 2017 +0530

--
 contrib/clients/python/lens/client/main.py |  4 +-
 contrib/clients/python/lens/client/query.py| 43 -
 contrib/clients/python/setup.py|  2 +
 contrib/clients/python/test/test_lensclient.py | 13 +--
 tools/conf/server/lens-site.xml|  6 +++
 5 files changed, 54 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/main.py
--
diff --git a/contrib/clients/python/lens/client/main.py 
b/contrib/clients/python/lens/client/main.py
index bf5d81e..1e9d88f 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -21,7 +21,8 @@ from .log import LensLogClient
 from .session import LensSessionClient
 from .query import LensQueryClient
 from .utils import xml_file_to_conf
-
+import logging
+logger = logging.getLogger(__name__)
 
 class LensClient(object):
 def __init__(self, base_url=None, username="", password="", database=None, 
conf=None):
@@ -35,6 +36,7 @@ class LensClient(object):
 self.base_url = base_url or conf.get('lens.server.base.url', 
"http://0.0.0.0:/lensapi;)
 if self.base_url[-1] != '/':
 self.base_url += "/"
+logger.debug("Using conf %s", conf)
 username = username or conf.get('lens.client.user.name', "anonymous")
 database = database or conf.get('lens.client.dbname')
 self.session = LensSessionClient(self.base_url, username, password, 
database, conf)

http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/query.py
--
diff --git a/contrib/clients/python/lens/client/query.py 
b/contrib/clients/python/lens/client/query.py
index f82f0cb..df16cca 100644
--- a/contrib/clients/python/lens/client/query.py
+++ b/contrib/clients/python/lens/client/query.py
@@ -15,21 +15,26 @@
 # limitations under the License.
 #
 import codecs
+import csv
+import logging
 import time
 import zipfile
 
 import requests
-from six import string_types, BytesIO, StringIO, PY2, PY3
+from requests.exceptions import HTTPError
+from six import string_types, BytesIO, PY2, PY3
+
 from .models import WrappedJson
 from .utils import conf_to_xml
-import csv
 
+logger = logging.getLogger(__name__)
 long_type = int
 
 if PY3:
 from collections.abc import Iterable as Iterable
 elif PY2:
 from collections import Iterable as Iterable
+
 long_type = long
 
 
@@ -70,6 +75,7 @@ type_mappings = {'BOOLEAN': bool,
  }
 default_mapping = lambda x: x
 
+
 class LensQueryResult(Iterable):
 def __init__(self, custom_mappings=None):
 if custom_mappings is None:
@@ -93,6 +99,7 @@ class LensInMemoryResult(LensQueryResult):
 for row in self.rows:
 yield list(self._mapping(value.type)(value.value) if value else 
None for value in row['values'])
 
+
 class LensPersistentResult(LensQueryResult):
 def __init__(self, header, response, encoding=None, 
is_header_present=True, delimiter=",",
  custom_mappings=None):
@@ -107,6 +114,11 @@ class LensPersistentResult(LensQueryResult):
 def _parse_line(self, line):
 return 
list(self._mapping(self.header.columns[index].type)(line[index]) for index in 
range(len(line)))
 
+def get_csv_reader(self, file):
+if PY3:
+file = codecs.iterdecode(file, 'utf-8')
+return csv.reader(file, delimiter=self.delimiter)
+
 def __iter__(self):
 if self.is_zipped:
 byte_stream = BytesIO(self.response.content)
@@ -114,7 +126,7 @@ class LensPersistentResult(LensQueryResult):
 for name in self.zipfile.namelist():
 with self.zipfile.open(name) as single_file:
 if name[-3:] == 'csv':
-reader = csv.reader(single_file, 
delimiter=self.delimiter)
+reader = self.get_csv_reader(single_file)
 else:
 reader = single_file
 reader_iterator = iter(reader)
@@ -142,7 +154,8 @@ 

[11/16] lens git commit: LENS-1386 : Add support for separate tables for update periods in one storage

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/f0dadd79/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 0e6a4a1..f6f6e77 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -62,7 +62,6 @@ import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 
 import org.glassfish.jersey.test.TestProperties;
-
 import org.testng.Assert;
 import org.testng.annotations.*;
 
@@ -974,7 +973,29 @@ public class TestMetastoreService extends LensJerseyTest {
 final String[] timePartColNames = {"dt"};
 return createStorageTblElement(storageName, table, timePartColNames, 
updatePeriod);
   }
+  private XStorageTableElement 
createStorageTblWithMultipleTableDescriptors(String storageName, String[] 
tables,
+ String [] updatePeriods) {
+String [][] timePartColNames  = new String[updatePeriods.length][];
+for (int i = 0; i < updatePeriods.length; i++) {
+  timePartColNames[i] = new String[]{ "dt" };
+}
+return createStorageTblWithMultipleTableDescriptors(storageName, tables, 
timePartColNames, updatePeriods);
+  }
 
+  private XStorageTableElement 
createStorageTblWithMultipleTableDescriptors(String storageName, String[] 
tables,
+String[][] timePartColNames, String [] updatePeriods) {
+XStorageTableElement tbl = cubeObjectFactory.createXStorageTableElement();
+tbl.setStorageName(storageName);
+XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
+tbl.setUpdatePeriods(xUpdatePeriods);
+for (int i = 0; i < updatePeriods.length; i++) {
+  XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new 
XUpdatePeriodTableDescriptor();
+  
updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(updatePeriods[i]));
+  
updatePeriodTableDescriptor.setTableDesc(createStorageTableDesc(tables[i], 
timePartColNames[i]));
+  
xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
+}
+return tbl;
+  }
   private XStorageTableElement createStorageTblElement(String storageName, 
String table,
 final String[] timePartColNames, String... updatePeriod) {
 XStorageTableElement tbl = cubeObjectFactory.createXStorageTableElement();
@@ -1836,6 +1857,169 @@ public class TestMetastoreService extends 
LensJerseyTest {
   }
 
   @Test(dataProvider = "mediaTypeData")
+  public void testCreateFactTableWithMultipleUpdatePeriods(MediaType 
mediaType) throws Exception {
+
+final String table = "testCreateFactTableWithMultipleUpdatePeriods";
+String prevDb = getCurrentDatabase(mediaType);
+final String DB = dbPFX + 
"testCreateFactTableWithMultipleUpdatePeriods_DB" + mediaType.getSubtype();
+createDatabase(DB, mediaType);
+setCurrentDatabase(DB, mediaType);
+createStorage("S1", mediaType);
+try {
+  final XCube cube = createTestCube("testCube");
+  target().path("metastore").path("cubes").queryParam("sessionid", 
lensSessionId).request(mediaType)
+.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXCube(cube)) {
+}, mediaType), APIResult.class);
+  XFactTable f = createFactTable(table);
+  String[] tables = new String[] { "testTable1", "testTable2", 
"testTable3" };
+  String[] updatePeriods = new String[] { "HOURLY", "DAILY", "MONTHLY" };
+  f.getStorageTables().getStorageTable()
+.add(createStorageTblWithMultipleTableDescriptors("S1", tables, 
updatePeriods));
+  APIResult result = 
target().path("metastore").path("facts").queryParam("sessionid", lensSessionId)
+.request(mediaType)
+.post(Entity.entity(new 
GenericEntity(cubeObjectFactory.createXFactTable(f)) {
+}, mediaType), APIResult.class);
+  assertSuccess(result);
+
+  StringList factNames = 
target().path("metastore/facts").queryParam("sessionid", 
lensSessionId).request(mediaType)
+.get(StringList.class);
+  assertTrue(factNames.getElements().contains(table.toLowerCase()));
+
+  // Get the created tables
+  JAXBElement gotFactElement = 
target().path("metastore/facts").path(table)
+.queryParam("sessionid", lensSessionId).request(mediaType).get(new 
GenericType() {
+});
+  XFactTable gotFact = gotFactElement.getValue();
+  assertTrue(gotFact.getName().equalsIgnoreCase(table));
+  assertEquals(gotFact.getWeight(), 10.0);
+
+  // Check for the created tables per update period.
+  List updatePeriodTableDescriptor = 
gotFact.getStorageTables().getStorageTable()
+

[03/16] lens git commit: LENS-1370 : Fix purging queries which are before retry framework

[prongs]

LENS-1370 : Fix purging queries which are before retry framework


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/5ac68804
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/5ac68804
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/5ac68804

Branch: refs/heads/lens-1381
Commit: 5ac688048ef2f8a233f022311dd8bbbd07aa0757
Parents: 4fb2506
Author: Rajat Khandelwal 
Authored: Wed Jan 18 17:49:07 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Wed Jan 18 17:49:07 2017 +0530

--
 .../main/java/org/apache/lens/server/query/LensServerDAO.java  | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/5ac68804/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java 
b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index dd489e8..cc6ca7d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -139,8 +139,10 @@ public class LensServerDAO {
 query.getDriverEndTime(), query.getDriverName(), 
query.getQueryName(), query.getSubmissionTime(),
 query.getDriverQuery(), serializeConf(query.getConf()),
 query.getFailedAttempts() == null ? 0 : 
query.getFailedAttempts().size());
-for (int i = 0; i < query.getFailedAttempts().size(); i++) {
-  insertFailedAttempt(runner, conn, query.getHandle(), 
query.getFailedAttempts().get(i), i);
+if (query.getFailedAttempts() != null) {
+  for (int i = 0; i < query.getFailedAttempts().size(); i++) {
+insertFailedAttempt(runner, conn, query.getHandle(), 
query.getFailedAttempts().get(i), i);
+  }
 }
 conn.commit();
   } finally {

[06/16] lens git commit: LENS-773: Include dimension table partitions in FactPartitionBasedQueryCostCalculator

[prongs]

LENS-773: Include dimension table partitions in 
FactPartitionBasedQueryCostCalculator


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6af57fb6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6af57fb6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6af57fb6

Branch: refs/heads/lens-1381
Commit: 6af57fb6748eaf2501258d1f9756c3476a6e3087
Parents: d46e78c
Author: Amareshwari Sriramadasu 
Authored: Mon Jan 30 12:42:14 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Mon Jan 30 12:42:14 2017 +0530

--
 .../FactPartitionBasedQueryCostCalculator.java  |  3 +++
 ...stFactPartitionBasedQueryCostCalculator.java | 21 +++-
 2 files changed, 19 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/6af57fb6/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
index 9fecdbc..3c157ee 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
@@ -67,6 +67,9 @@ public class FactPartitionBasedQueryCostCalculator implements 
QueryCostCalculato
   }
   cost += allTableWeights * 
getNormalizedUpdatePeriodCost(partition.getPeriod(), driver);
 }
+  } else {
+// increase cost for every dimtable partition
+cost += 1.0;
   }
 }
 return cost;

http://git-wip-us.apache.org/repos/asf/lens/blob/6af57fb6/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
 
b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
index 262d452..b5e73fd 100644
--- 
a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
+++ 
b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
@@ -25,6 +25,7 @@ import static org.apache.lens.cube.metadata.UpdatePeriod.*;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.*;
 
 import java.util.HashMap;
 import java.util.Set;
@@ -43,12 +44,12 @@ import org.testng.annotations.Test;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
-import junit.framework.Assert;
 
 public class TestFactPartitionBasedQueryCostCalculator {
   AbstractQueryContext queryContext;
   FactPartitionBasedQueryCostCalculator calculator = new 
FactPartitionBasedQueryCostCalculator();
   LensDriver driver;
+  private static String latest = "latest";
 
   @BeforeTest
   public void setUp() {
@@ -68,6 +69,7 @@ public class TestFactPartitionBasedQueryCostCalculator {
 HashMap partitions = new HashMap<>();
 partitions.put("st1", Sets.newHashSet(fp1, fp2));
 partitions.put("st2", Sets.newHashSet(fp3, fp4));
+partitions.put("st3", Sets.newHashSet(latest));
 DriverQueryPlan plan = mock(DriverQueryPlan.class);
 when(queryContext.getDriverRewriterPlan(driver)).thenReturn(plan);
 when(plan.getPartitions()).thenReturn(partitions);
@@ -85,12 +87,21 @@ public class TestFactPartitionBasedQueryCostCalculator {
   @Test
   public void testCalculateCost() throws Exception {
 QueryCost cost = calculator.calculateCost(queryContext, driver);
-Assert.assertTrue(cost.getEstimatedResourceUsage() > 18.0);
-Assert.assertTrue(cost.getEstimatedResourceUsage() < 19.0);
+assertTrue(cost.getEstimatedResourceUsage() > 19.0, "Estimated resource 
usage:" + cost.getEstimatedResourceUsage());
+assertTrue(cost.getEstimatedResourceUsage() < 20.0, "Estimated resource 
usage:" + cost.getEstimatedResourceUsage());
   }
 
   @Test
-  public void testGetAllPartitions() throws Exception {
-
+  public void testDimensionCost() throws Exception {
+AbstractQueryContext queryContext2 = mock(AbstractQueryContext.class);
+HashMap partitions = new HashMap<>();
+partitions.put("st1", Sets.newHashSet(latest));
+partitions.put("st2", Sets.newHashSet(latest));
+DriverQueryPlan plan = mock(DriverQueryPlan.class);
+

[15/16] lens git commit: LENS-1398(incremental): Python client not raising exception in some cases

[prongs]

LENS-1398(incremental): Python client not raising exception in some cases


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d1b43d61
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d1b43d61
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d1b43d61

Branch: refs/heads/lens-1381
Commit: d1b43d615ce950609d0496ad8db1a63813f3042b
Parents: a3045fe
Author: Rajat Khandelwal 
Authored: Mon Mar 27 14:42:38 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Mar 27 14:42:38 2017 +0530

--
 contrib/clients/python/lens/client/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d1b43d61/contrib/clients/python/lens/client/main.py
--
diff --git a/contrib/clients/python/lens/client/main.py 
b/contrib/clients/python/lens/client/main.py
index 1e9d88f..b1846ad 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -25,7 +25,8 @@ import logging
 logger = logging.getLogger(__name__)
 
 class LensClient(object):
-def __init__(self, base_url=None, username="", password="", database=None, 
conf=None):
+def __init__(self, base_url=None, username="", password="", database=None, 
conf=None, logging_level=logging.INFO):
+logging.basicConfig(level=logging_level)
 if conf and isinstance(conf, string_types) and os.path.exists(conf):
 if os.path.isdir(conf):
 conf = os.path.join(conf, 'lens-client-site.xml')

[13/16] lens git commit: LENS-1390 : Fix issues with updatePeriodTableDescriptor

[prongs]

LENS-1390 : Fix issues with updatePeriodTableDescriptor


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bd344c7f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bd344c7f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bd344c7f

Branch: refs/heads/lens-1381
Commit: bd344c7ff0f7f3eebe16ca9457d2fed2f29b61da
Parents: f0dadd7
Author: Lavkesh Lahngir 
Authored: Tue Feb 21 09:31:38 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Tue Feb 21 09:31:38 2017 +0530

--
 lens-api/src/main/resources/cube-0.1.xsd | 6 +++---
 lens-examples/src/test/resources/yaml/city_table.yaml| 3 ++-
 lens-examples/src/test/resources/yaml/customer_table.yaml| 3 ++-
 lens-examples/src/test/resources/yaml/dim_table.yaml | 3 ++-
 lens-examples/src/test/resources/yaml/dim_table2.yaml| 3 ++-
 lens-examples/src/test/resources/yaml/dim_table4.yaml| 3 ++-
 lens-examples/src/test/resources/yaml/fact1.yaml | 3 ++-
 lens-examples/src/test/resources/yaml/fact2.yaml | 3 ++-
 lens-examples/src/test/resources/yaml/product_table.yaml | 3 ++-
 lens-examples/src/test/resources/yaml/rawfact.yaml   | 3 ++-
 .../src/test/resources/yaml/sales-aggr-continuous-fact.yaml  | 3 ++-
 lens-examples/src/test/resources/yaml/sales-aggr-fact1.yaml  | 6 --
 lens-examples/src/test/resources/yaml/sales-aggr-fact2.yaml  | 6 --
 lens-examples/src/test/resources/yaml/sales-raw-fact.yaml| 3 ++-
 .../java/org/apache/lens/server/metastore/JAXBUtils.java | 8 +---
 15 files changed, 38 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/bd344c7f/lens-api/src/main/resources/cube-0.1.xsd
--
diff --git a/lens-api/src/main/resources/cube-0.1.xsd 
b/lens-api/src/main/resources/cube-0.1.xsd
index 431d68b..060eb43 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -1024,11 +1024,11 @@
 of update_period_table_descriptor instead of a list of enums.
   
 
-
-  
+
+  
   
   
-
+
   
 
   

http://git-wip-us.apache.org/repos/asf/lens/blob/bd344c7f/lens-examples/src/test/resources/yaml/city_table.yaml
--
diff --git a/lens-examples/src/test/resources/yaml/city_table.yaml 
b/lens-examples/src/test/resources/yaml/city_table.yaml
index 4f3b986..92ce3ec 100644
--- a/lens-examples/src/test/resources/yaml/city_table.yaml
+++ b/lens-examples/src/test/resources/yaml/city_table.yaml
@@ -22,7 +22,8 @@ properties:
   city.prop: d1
 storageTables:
   local:
-updatePeriods: HOURLY
+updatePeriods:
+  updatePeriod: HOURLY
 tableDesc:
   partCols:
 dt: type: STRING, comment: Time column

http://git-wip-us.apache.org/repos/asf/lens/blob/bd344c7f/lens-examples/src/test/resources/yaml/customer_table.yaml
--
diff --git a/lens-examples/src/test/resources/yaml/customer_table.yaml 
b/lens-examples/src/test/resources/yaml/customer_table.yaml
index 2de5cd9..4209bea 100644
--- a/lens-examples/src/test/resources/yaml/customer_table.yaml
+++ b/lens-examples/src/test/resources/yaml/customer_table.yaml
@@ -24,7 +24,8 @@ properties:
   dim4.prop: d1
 storageTables:
   local:
-updatePeriods: HOURLY
+updatePeriods:
+  updatePeriod: HOURLY
 tableDesc:
   partCols:
 dt: type: STRING, comment: Time column

http://git-wip-us.apache.org/repos/asf/lens/blob/bd344c7f/lens-examples/src/test/resources/yaml/dim_table.yaml
--
diff --git a/lens-examples/src/test/resources/yaml/dim_table.yaml 
b/lens-examples/src/test/resources/yaml/dim_table.yaml
index 8191456..37b50da 100644
--- a/lens-examples/src/test/resources/yaml/dim_table.yaml
+++ b/lens-examples/src/test/resources/yaml/dim_table.yaml
@@ -22,7 +22,8 @@ properties:
   dim1.prop: d1
 storageTables:
   local:
-updatePeriods: HOURLY
+updatePeriods:
+  updatePeriod: HOURLY
 tableDesc:
   partCols:
 dt: type: STRING, comment: Time column

http://git-wip-us.apache.org/repos/asf/lens/blob/bd344c7f/lens-examples/src/test/resources/yaml/dim_table2.yaml
--
diff --git a/lens-examples/src/test/resources/yaml/dim_table2.yaml 
b/lens-examples/src/test/resources/yaml/dim_table2.yaml
index 0df4682..619625c 100644
--- a/lens-examples/src/test/resources/yaml/dim_table2.yaml
+++ b/lens-examples/src/test/resources/yaml/dim_table2.yaml
@@ -21,7 +21,8 @@ properties:
   

[02/16] lens git commit: LENS-1378 : Fix dimensions to query from DenormalizationResolver for expression fields

[prongs]

LENS-1378 : Fix dimensions to query from DenormalizationResolver for expression 
fields


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fb25061
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fb25061
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fb25061

Branch: refs/heads/lens-1381
Commit: 4fb2506193acd3efa7d90bb640bd50e2aefe5d6a
Parents: 98990c3
Author: Amareshwari Sriramadasu 
Authored: Wed Jan 18 17:46:58 2017 +0530
Committer: Amareshwari Sriramadasu 
Committed: Wed Jan 18 17:46:58 2017 +0530

--
 .../lens/cube/parse/CandidateTableResolver.java |   2 +-
 .../lens/cube/parse/CubeQueryContext.java   |  11 +-
 .../cube/parse/DenormalizationResolver.java | 167 +--
 .../lens/cube/parse/ExpressionResolver.java | 107 ++--
 .../lens/cube/parse/QueriedPhraseContext.java   |   2 +-
 .../lens/cube/parse/TimeRangeChecker.java   |  18 --
 .../lens/cube/parse/TrackDenormContext.java |  37 
 .../apache/lens/cube/parse/CubeTestSetup.java   |  11 +-
 .../lens/cube/parse/TestExpressionResolver.java |  15 ++
 .../lens/cube/parse/TestJoinResolver.java   |   2 +-
 tools/conf/server/logback.xml   |   4 +-
 11 files changed, 248 insertions(+), 128 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index e7fc557..e9270ea 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -697,7 +697,7 @@ class CandidateTableResolver implements ContextRewriter {
 i.remove();
 break;
   }
-} else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, 
dim.getName())) {
+} else if (!cubeql.getDeNormCtx().addRefUsage(cubeql, cdim, 
col, dim.getName())) {
   // check if it available as reference, if not remove the
   // candidate
   log.info("Not considering dimtable: {} as column {} is not 
available", cdim, col);

http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index e83ae76..125b432 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -59,7 +59,7 @@ import lombok.*;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class CubeQueryContext extends TracksQueriedColumns implements QueryAST 
{
+public class CubeQueryContext extends TracksQueriedColumns implements 
QueryAST, TrackDenormContext {
   public static final String TIME_RANGE_FUNC = "time_range_in";
   public static final String NOW = "now";
   public static final String DEFAULT_TABLE = "_default_";
@@ -922,7 +922,8 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST {
 Set exprDimensions = new HashSet<>();
 if (cfacts != null) {
   for (CandidateFact cfact : cfacts) {
-Set factExprDimTables = exprCtx.rewriteExprCtx(cfact, 
dimsToQuery, cfacts.size() > 1 ? cfact : this);
+Set factExprDimTables = exprCtx.rewriteExprCtx(this, cfact, 
dimsToQuery,
+  cfacts.size() > 1 ? cfact : this);
 exprDimensions.addAll(factExprDimTables);
 if (cfacts.size() > 1) {
   factDimMap.get(cfact).addAll(factExprDimTables);
@@ -933,7 +934,7 @@ public class CubeQueryContext extends TracksQueriedColumns 
implements QueryAST {
   }
 } else {
   // dim only query
-  exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
+  exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, 
this));
 }
 dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
 log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
@@ -942,14 +943,14 @@ public class CubeQueryContext extends 
TracksQueriedColumns implements QueryAST {
 Set denormTables = new HashSet<>();
 if (cfacts != null) {
   for (CandidateFact cfact : cfacts) {
-Set factDenormTables = deNormCtx.rewriteDenormctx(cfact, 
dimsToQuery, 

lens git commit: LENS-1403 : Measures getting repeated in inner select of union query

[prongs]

Repository: lens
Updated Branches:
  refs/heads/lens-1381 363f132d1 -> d45c5384c


LENS-1403 : Measures getting repeated in inner select of union query


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d45c5384
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d45c5384
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d45c5384

Branch: refs/heads/lens-1381
Commit: d45c5384ccab119aa263e3bc4b2c3a6c78f8c993
Parents: 363f132
Author: Sushil Mohanty 
Authored: Wed Mar 29 15:24:04 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Mar 29 15:24:04 2017 +0530

--
 .../lens/cube/parse/StorageCandidate.java   |  2 +
 .../lens/cube/parse/UnionQueryWriter.java   | 20 +---
 .../lens/cube/parse/TestBaseCubeQueries.java| 30 +--
 .../cube/parse/TestUnionAndJoinCandidates.java  | 53 
 4 files changed, 63 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 17f3af8..628e9aa 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -340,6 +340,8 @@ public class StorageCandidate implements Candidate, 
CandidateTable {
 if (this == cubeql.getPickedCandidate()) {
   CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql);
   updateOrderByWithFinalAlias(queryAst.getOrderByAST(), 
queryAst.getSelectAST());
+} else {
+  queryAst.setHavingAST(null);
 }
 return CandidateUtil
   .buildHQLString(queryAst.getSelectString(), fromString, whereString, 
queryAst.getGroupByString(),

http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f9717fa..f2325f1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -142,7 +142,11 @@ public class UnionQueryWriter {
 ASTNode expr = innerToOuterSelectASTs.containsKey(new 
HQLParser.HashableASTNode(node))
 ? innerToOuterSelectASTs.get(new HQLParser.HashableASTNode(node))
 : innerToOuterHavingASTs.get(new HQLParser.HashableASTNode(node));
-node.getParent().setChild(0, expr);
+if (node.getChildCount() > 1) {
+  node.replaceChildren(1, 1, expr.getChild(1));
+} else {
+  node.replaceChildren(0, 0, expr);
+}
   }
 }
 for (int i = 0; i < node.getChildCount(); i++) {
@@ -191,7 +195,7 @@ public class UnionQueryWriter {
   ASTNode outerOrderby = new ASTNode(child);
   ASTNode tokNullsChild = (ASTNode) child.getChild(0);
   ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
-  outerTokNullsChild.addChild(getOuterAST((ASTNode) 
tokNullsChild.getChild(0), null, aliasDecider, null, true, 
cubeql.getBaseCube().getDimAttributeNames()));
+  outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new 
HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
   outerOrderby.addChild(outerTokNullsChild);
   outerExpression.addChild(outerOrderby);
 }
@@ -299,8 +303,7 @@ public class UnionQueryWriter {
 
   private List getProjectedNonDefaultPhrases() {
 List phrases = new ArrayList<>();
-int selectPhraseCount = cubeql.getSelectPhrases().size();
-for (int i = 0; i < selectPhraseCount; i++) {
+for (int i = 0; i < 
storageCandidates.iterator().next().getQueryAst().getSelectAST().getChildCount();
 i++) {
   for (StorageCandidate sc : storageCandidates) {
 ASTNode selectAST = sc.getQueryAst().getSelectAST();
 if (isNodeDefault((ASTNode) selectAST.getChild(i))) {
@@ -359,10 +362,13 @@ public class UnionQueryWriter {
 }
   }
 }
-updateOuterSelectDuplicateAliases(queryAst.getSelectAST(), aliasMap);
+updateOuterASTDuplicateAliases(queryAst.getSelectAST(), aliasMap);
+if (queryAst.getHavingAST() != null) {
+  updateOuterASTDuplicateAliases(queryAst.getHavingAST(), aliasMap);
+}
   }
 
-  public void updateOuterSelectDuplicateAliases(ASTNode node,
+  public void updateOuterASTDuplicateAliases(ASTNode node,
   

[2/2] lens git commit: LENS-1397: Support query rewrite for separate table per update period in a storage

[prongs]

LENS-1397: Support query rewrite for separate table per update period in a 
storage


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/363f132d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/363f132d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/363f132d

Branch: refs/heads/lens-1381
Commit: 363f132d140b5107bc2afbc3c75cb9b54bf64a65
Parents: 112af59
Author: Puneet Gupta 
Authored: Wed Mar 29 15:21:26 2017 +0530
Committer: Rajat Khandelwal 
Committed: Wed Mar 29 15:21:26 2017 +0530

--
 .../lens/cube/metadata/CubeMetastoreClient.java |  30 +-
 .../apache/lens/cube/metadata/TimeRange.java|   2 -
 .../org/apache/lens/cube/parse/Candidate.java   |  17 +-
 .../parse/CandidateCoveringSetsResolver.java|   3 -
 .../cube/parse/CandidateTablePruneCause.java|   4 +-
 .../lens/cube/parse/CandidateTableResolver.java |   1 -
 .../apache/lens/cube/parse/CandidateUtil.java   |  39 +-
 .../lens/cube/parse/ColumnLifetimeChecker.java  |   6 -
 .../lens/cube/parse/CubeQueryContext.java   | 133 --
 .../lens/cube/parse/CubeSemanticAnalyzer.java   |   1 -
 .../apache/lens/cube/parse/DefaultQueryAST.java |   2 +
 .../cube/parse/DenormalizationResolver.java |   3 +-
 .../lens/cube/parse/ExpressionResolver.java |   2 +-
 .../apache/lens/cube/parse/JoinCandidate.java   |   6 +
 .../cube/parse/MaxCoveringFactResolver.java |   4 +-
 .../org/apache/lens/cube/parse/QueryAST.java|   4 +
 .../lens/cube/parse/StorageCandidate.java   | 475 ++-
 .../lens/cube/parse/StorageTableResolver.java   |  52 +-
 .../apache/lens/cube/parse/UnionCandidate.java  |  11 +
 .../lens/cube/parse/UnionQueryWriter.java   |   9 +-
 .../lens/cube/parse/join/AutoJoinContext.java   |   2 +-
 .../apache/lens/cube/metadata/DateFactory.java  |  29 +-
 .../apache/lens/cube/parse/CubeTestSetup.java   |  12 +-
 .../lens/cube/parse/TestBaseCubeQueries.java|  17 +-
 .../cube/parse/TestDenormalizationResolver.java |  16 +-
 .../lens/cube/parse/TestQueryMetrics.java   |   7 +-
 .../lens/cube/parse/TestTimeRangeResolver.java  |   6 +-
 .../parse/TestTimeRangeWriterWithQuery.java |   4 +-
 .../lens/cube/parse/TestUnionQueries.java   |  44 ++
 .../test/resources/schema/facts/testfact.xml|  39 ++
 30 files changed, 738 insertions(+), 242 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/363f132d/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index c8a2498..7608a43 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -2261,17 +2261,37 @@ public class CubeMetastoreClient {
 return dimTables;
   }
 
-  public boolean partColExists(String tableName, String partCol) throws 
LensException {
-Table tbl = getTable(tableName);
-for (FieldSchema f : tbl.getPartCols()) {
-  if (f.getName().equalsIgnoreCase(partCol)) {
-return true;
+  public boolean partColExists(String fact, String storage, String partCol) 
throws LensException {
+for (String storageTable : getStorageTables(fact, storage)) {
+  for (FieldSchema f : getTable(storageTable).getPartCols()) {
+if (f.getName().equalsIgnoreCase(partCol)) {
+  return true;
+}
   }
 }
 return false;
   }
 
   /**
+   * Returns storage table names for a storage.
+   * Note: If each update period in the storage has a different storage table, 
this method will return N Storage Tables
+   * where N is the number of update periods in the storage (LENS-1386)
+   *
+   * @param fact
+   * @param storage
+   * @return
+   * @throws LensException
+   */
+  public Set getStorageTables(String fact, String storage) throws 
LensException {
+Set uniqueStorageTables = new HashSet<>();
+for (UpdatePeriod updatePeriod : 
getFactTable(fact).getUpdatePeriods().get(storage)) {
+  uniqueStorageTables.add(getStorageTableName(fact, storage, 
updatePeriod));
+}
+return uniqueStorageTables;
+  }
+
+
+  /**
*
* @param table table name
* @param hiveTable hive table

http://git-wip-us.apache.org/repos/asf/lens/blob/363f132d/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java 

[1/2] lens git commit: LENS-1397: Support query rewrite for separate table per update period in a storage

[prongs]

Repository: lens
Updated Branches:
  refs/heads/lens-1381 112af59cc -> 363f132d1


http://git-wip-us.apache.org/repos/asf/lens/blob/363f132d/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index 523a876..7874a66 100644
--- 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -21,7 +21,7 @@ package org.apache.lens.cube.parse;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static com.google.common.collect.Sets.newHashSet;
-import static java.util.stream.Collectors.toMap;
+
 import static org.apache.lens.cube.metadata.DateFactory.*;
 import static 
org.apache.lens.cube.parse.CandidateTablePruneCause.columnNotFound;
 import static org.apache.lens.cube.parse.CubeTestSetup.*;
@@ -193,21 +193,21 @@ public class TestDenormalizationResolver extends 
TestQueryRewrite {
 LensException e = getLensExceptionInRewrite(
   "select dim2big2, max(msr3)," + " msr2 from testCube" + " where " + 
TWO_DAYS_RANGE, tconf);
 NoCandidateFactAvailableException ne = (NoCandidateFactAvailableException) 
e;
-PruneCauses.BriefAndDetailedError error = ne.getJsonMessage(); // Storage 
update periods are not valid for given time range
+PruneCauses.BriefAndDetailedError error = ne.getJsonMessage();
 Assert.assertEquals(error.getBrief(), 
CandidateTablePruneCode.UNSUPPORTED_STORAGE.errorFormat);
 
 Map enhanced = 
error.enhanced();
 Map expected = 
Maps.newHashMap();
-expected.put(newHashSet("c1_summary1","c1_testfact","c1_testfact2"),
+expected.put(newHashSet("c1_summary1", "c1_testfact", "c1_testfact2"),
   newArrayList(columnNotFound("dim2big2")));
-expected.put(newHashSet("c2_summary2","c2_summary3","c1_testfact2_raw",""
-  + "c3_testfact2_raw","c1_summary3","c1_summary2"),
+expected.put(newHashSet("c2_summary2", "c2_summary3", "c1_testfact2_raw", 
""
++ "c3_testfact2_raw", "c1_summary3", "c1_summary2"),
   newArrayList(new 
CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE)));
 expected.put(newHashSet("c0_testfact_continuous"), 
newArrayList(columnNotFound(
   "msr2", "msr3")));
-
expected.put(newHashSet("c2_summary2","c2_summary3","c2_summary4","c4_testfact","c2_summary1",
-  
"c3_testfact","c3_testfact2_raw","c4_testfact2","c5_testfact","c99_cheapfact","c2_testfact","c0_cheapfact",
-  "c2_testfactmonthly","c0_testfact"),
+expected.put(newHashSet("c2_summary2", "c2_summary3", "c2_summary4", 
"c4_testfact", "c2_summary1",
+  "c3_testfact", "c3_testfact2_raw", "c6_testfact", "c4_testfact2", 
"c5_testfact", "c99_cheapfact",
+  "c2_testfact", "c0_cheapfact", "c2_testfactmonthly", "c0_testfact"),
   newArrayList(new 
CandidateTablePruneCause(CandidateTablePruneCode.UNSUPPORTED_STORAGE)));
 
 Assert.assertEquals(enhanced, expected);

http://git-wip-us.apache.org/repos/asf/lens/blob/363f132d/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
--
diff --git 
a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java 
b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
index 46ee863..3883bee 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
@@ -21,8 +21,6 @@ package org.apache.lens.cube.parse;
 
 import static org.apache.lens.cube.metadata.DateFactory.TWO_DAYS_RANGE;
 
-import java.util.Arrays;
-
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.metrics.LensMetricsRegistry;
 
@@ -59,7 +57,8 @@ public class TestQueryMetrics extends TestQueryRewrite {
   
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.ColumnLifetimeChecker-ITER-10",
   
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-11",
   
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateTableResolver-ITER-12",
-  
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.CandidateCoveringSetsResolver-ITER-13",
+  
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse."
++ "CandidateCoveringSetsResolver-ITER-13",
   
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-14",
   
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-15",
 

lens git commit: LENS-1398(incremental): Python client not raising exception in some cases

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master a3045fee1 -> d1b43d615


LENS-1398(incremental): Python client not raising exception in some cases


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d1b43d61
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d1b43d61
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d1b43d61

Branch: refs/heads/master
Commit: d1b43d615ce950609d0496ad8db1a63813f3042b
Parents: a3045fe
Author: Rajat Khandelwal 
Authored: Mon Mar 27 14:42:38 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Mar 27 14:42:38 2017 +0530

--
 contrib/clients/python/lens/client/main.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/d1b43d61/contrib/clients/python/lens/client/main.py
--
diff --git a/contrib/clients/python/lens/client/main.py 
b/contrib/clients/python/lens/client/main.py
index 1e9d88f..b1846ad 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -25,7 +25,8 @@ import logging
 logger = logging.getLogger(__name__)
 
 class LensClient(object):
-def __init__(self, base_url=None, username="", password="", database=None, 
conf=None):
+def __init__(self, base_url=None, username="", password="", database=None, 
conf=None, logging_level=logging.INFO):
+logging.basicConfig(level=logging_level)
 if conf and isinstance(conf, string_types) and os.path.exists(conf):
 if os.path.isdir(conf):
 conf = os.path.join(conf, 'lens-client-site.xml')

[1/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup using xml files instead of code

[prongs]

Repository: lens
Updated Branches:
  refs/heads/lens-1381 de464faa8 -> 112af59cc


http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java 
b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
deleted file mode 100644
index 7d54c7b..000
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ /dev/null
@@ -1,1116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.metastore;
-
-import java.lang.reflect.Constructor;
-import java.text.ParseException;
-import java.util.*;
-
-import javax.ws.rs.WebApplicationException;
-import javax.xml.datatype.DatatypeConfigurationException;
-import javax.xml.datatype.DatatypeFactory;
-import javax.xml.datatype.XMLGregorianCalendar;
-
-import org.apache.lens.api.metastore.*;
-import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
-import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
-import org.apache.lens.server.api.error.LensException;
-
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.mapred.InputFormat;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.Maps;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * Utilities for converting to and from JAXB types to hive.ql.metadata.cube 
types
- */
-@Slf4j
-public final class JAXBUtils {
-  private JAXBUtils() {
-
-  }
-
-  private static final ObjectFactory XCF = new ObjectFactory();
-
-  /**
-   * Create a hive ql cube object from corresponding JAXB object
-   *
-   * @param cube JAXB Cube
-   * @return {@link Cube}
-   * @throws LensException
-   */
-  public static CubeInterface hiveCubeFromXCube(XCube cube, Cube parent) 
throws LensException {
-if (cube instanceof XDerivedCube) {
-  XDerivedCube dcube = (XDerivedCube) cube;
-  Set dims = new LinkedHashSet();
-  dims.addAll(dcube.getDimAttrNames().getAttrName());
-
-  Set measures = new LinkedHashSet();
-  measures.addAll(dcube.getMeasureNames().getMeasureName());
-
-  Map properties = 
mapFromXProperties(cube.getProperties());
-  return new DerivedCube(cube.getName(), measures, dims, properties, 0L, 
parent);
-} else {
-  XBaseCube bcube = (XBaseCube) cube;
-  Set dims = new LinkedHashSet();
-  if (bcube.getDimAttributes() != null && 
!bcube.getDimAttributes().getDimAttribute().isEmpty()) {
-for (XDimAttribute xd : bcube.getDimAttributes().getDimAttribute()) {
-  dims.add(hiveDimAttrFromXDimAttr(xd));
-}
-  }
-
-  Set measures = new LinkedHashSet();
-  for (XMeasure xm : bcube.getMeasures().getMeasure()) {
-measures.add(hiveMeasureFromXMeasure(xm));
-  }
-
-  Set expressions = new LinkedHashSet();
-  if (bcube.getExpressions() != null && 
!bcube.getExpressions().getExpression().isEmpty()) {
-for (XExprColumn xe : bcube.getExpressions().getExpression()) {
-  expressions.add(hiveExprColumnFromXExprColumn(xe));
-}
-  }
-
-  Set joinchains = new LinkedHashSet();
-  if (bcube.getJoinChains() != null && 
!bcube.getJoinChains().getJoinChain().isEmpty()) {
-for (XJoinChain xj : bcube.getJoinChains().getJoinChain()) {
-  joinchains.add(joinChainFromXJoinChain(xj));
-}
-  }
-
-  Map properties = 
mapFromXProperties(cube.getProperties());
-  return new Cube(cube.getName(), measures, dims, expressions, joinchains, 
properties, 0L);
-}
-  }
-
-  /**
-   * Get XCube from hive.ql.metadata.Cube
-   *
-   * 

[2/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup using xml files instead of code

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact6_base.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/testfact6_base.xml 
b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml
new file mode 100644
index 000..42715e9
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact6_base.xml
@@ -0,0 +1,117 @@
+
+
+  
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+
+
+  
+  
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C3
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C4
+  
+
+  
+  
+
+
+  
+
+
+  
+
+ttd
+ttd2
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C1
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C2
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml 
b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml
new file mode 100644
index 000..94fb68a
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact_continuous.xml
@@ -0,0 +1,38 @@
+
+
+  
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+  
+  
+
+  
+CONTINUOUS
+  
+  C0
+  
+
+
+  
+  
+
+
+  
+
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml 
b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml
new file mode 100644
index 000..f14395e
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfact_deprecated.xml
@@ -0,0 +1,126 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C3
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C4
+  
+
+  
+  
+
+
+  
+
+
+  
+
+ttd
+ttd2
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C1
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C2
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml 
b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml
new file mode 100644
index 000..8237ba0
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/testfactmonthly.xml
@@ -0,0 +1,46 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+  
+  
+
+  
+MONTHLY
+  
+  C2
+  
+
+  
+
+
+  
+
+
+  
+
+

[4/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup using xml files instead of code

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/countrydim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/countrydim.xml 
b/lens-cube/src/test/resources/schema/dimensions/countrydim.xml
new file mode 100644
index 000..1e95416
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/countrydim.xml
@@ -0,0 +1,21 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml 
b/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml
new file mode 100644
index 000..a9cc3ae
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/cycledim1.xml
@@ -0,0 +1,51 @@
+
+
+  
+
+
+
+
+
+
+  
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml 
b/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml
new file mode 100644
index 000..b714f50
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/cycledim2.xml
@@ -0,0 +1,50 @@
+
+
+  
+
+
+
+
+
+
+  
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/daydim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/daydim.xml 
b/lens-cube/src/test/resources/schema/dimensions/daydim.xml
new file mode 100644
index 000..0ba6cee
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/daydim.xml
@@ -0,0 +1,41 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/hourdim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/hourdim.xml 
b/lens-cube/src/test/resources/schema/dimensions/hourdim.xml
new file mode 100644
index 000..c7bf7fb
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/hourdim.xml
@@ -0,0 +1,25 @@
+
+
+  
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/sports.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/sports.xml 
b/lens-cube/src/test/resources/schema/dimensions/sports.xml
new file mode 100644
index 000..d237069
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/sports.xml
@@ -0,0 +1,25 @@
+
+
+  
+
+
+
+
+  
+  
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/statedim.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimensions/statedim.xml 
b/lens-cube/src/test/resources/schema/dimensions/statedim.xml
new file mode 100644
index 000..ab55bdf
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimensions/statedim.xml
@@ -0,0 +1,53 @@
+
+
+  
+
+
+
+
+
+
+
+
+  
+  
+  
+
+  
+
+  
+
+  
+  
+
+  
+
+  
+
+  
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimensions/testdim2.xml

[7/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup using xml files instead of code

[prongs]

LENS-1400: Convert CubeTestSetup to setup using xml files instead of code


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/112af59c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/112af59c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/112af59c

Branch: refs/heads/lens-1381
Commit: 112af59cc4b82402a3df2c7b6687d2ee328e96b4
Parents: de464fa
Author: Rajat Khandelwal 
Authored: Mon Mar 27 14:36:58 2017 +0530
Committer: Rajat Khandelwal 
Committed: Mon Mar 27 14:36:58 2017 +0530

--
 .../java/org/apache/lens/api/ToXMLString.java   |9 +
 .../apache/lens/api/jaxb/LensJAXBContext.java   |   14 +-
 .../lens/api/metastore/SchemaTraverser.java |   58 +
 lens-api/src/main/resources/cube-0.1.xsd|4 +-
 .../lens/cli/commands/LensSchemaCommands.java   |  111 +-
 .../lens/cube/metadata/CubeFactTable.java   |   13 +-
 .../lens/cube/metadata/CubeMetastoreClient.java |  195 +-
 .../apache/lens/cube/metadata/JAXBUtils.java| 1114 
 .../lens/cube/parse/StorageCandidate.java   |8 +-
 .../lens/cube/parse/StorageTableResolver.java   |2 +-
 .../apache/lens/cube/parse/CubeTestSetup.java   | 2630 +-
 .../resources/schema/cubes/base/basecube.xml|  952 +++
 .../resources/schema/cubes/base/testcube.xml|  640 +
 .../resources/schema/cubes/derived/der1.xml |   22 +
 .../resources/schema/cubes/derived/der2.xml |   43 +
 .../resources/schema/cubes/derived/der3.xml |   32 +
 .../schema/cubes/derived/derivedcube.xml|   24 +
 .../cubes/derived/union_join_ctx_der1.xml   |   25 +
 .../resources/schema/dimensions/citydim.xml |  102 +
 .../resources/schema/dimensions/countrydim.xml  |   21 +
 .../resources/schema/dimensions/cycledim1.xml   |   51 +
 .../resources/schema/dimensions/cycledim2.xml   |   50 +
 .../test/resources/schema/dimensions/daydim.xml |   41 +
 .../resources/schema/dimensions/hourdim.xml |   25 +
 .../test/resources/schema/dimensions/sports.xml |   25 +
 .../resources/schema/dimensions/statedim.xml|   53 +
 .../resources/schema/dimensions/testdim2.xml|  203 ++
 .../resources/schema/dimensions/testdim3.xml|   48 +
 .../resources/schema/dimensions/testdim4.xml|   25 +
 .../schema/dimensions/unreachabledim.xml|   24 +
 .../schema/dimensions/user_interests.xml|   30 +
 .../resources/schema/dimensions/userdim.xml |   58 +
 .../test/resources/schema/dimensions/zipdim.xml |   30 +
 .../resources/schema/dimtables/citytable.xml|   51 +
 .../resources/schema/dimtables/citytable2.xml   |   29 +
 .../resources/schema/dimtables/citytable3.xml   |   29 +
 .../resources/schema/dimtables/citytable4.xml   |   28 +
 .../resources/schema/dimtables/countrytable.xml |   32 +
 .../dimtables/countrytable_partitioned.xml  |   35 +
 .../resources/schema/dimtables/cycledim1tbl.xml |   49 +
 .../resources/schema/dimtables/cycledim2tbl.xml |   49 +
 .../resources/schema/dimtables/daydimtbl.xml|   48 +
 .../resources/schema/dimtables/hourdimtbl.xml   |   48 +
 .../resources/schema/dimtables/sports_tbl.xml   |   48 +
 .../resources/schema/dimtables/statetable.xml   |   36 +
 .../schema/dimtables/statetable_partitioned.xml |   37 +
 .../resources/schema/dimtables/testdim2tbl.xml  |   50 +
 .../resources/schema/dimtables/testdim2tbl2.xml |   69 +
 .../resources/schema/dimtables/testdim2tbl3.xml |   69 +
 .../resources/schema/dimtables/testdim3tbl.xml  |   49 +
 .../resources/schema/dimtables/testdim4tbl.xml  |   48 +
 .../schema/dimtables/unreachabledimtable.xml|   35 +
 .../schema/dimtables/user_interests_tbl.xml |   50 +
 .../resources/schema/dimtables/usertable.xml|   52 +
 .../resources/schema/dimtables/ziptable.xml |   35 +
 .../test/resources/schema/facts/cheapfact.xml   |   81 +
 .../test/resources/schema/facts/summary1.xml|   81 +
 .../test/resources/schema/facts/summary2.xml|   83 +
 .../test/resources/schema/facts/summary3.xml|   84 +
 .../test/resources/schema/facts/summary4.xml|   65 +
 .../test/resources/schema/facts/testfact.xml|  192 ++
 .../resources/schema/facts/testfact1_base.xml   |  135 +
 .../schema/facts/testfact1_raw_base.xml |   75 +
 .../test/resources/schema/facts/testfact2.xml   |   75 +
 .../resources/schema/facts/testfact2_base.xml   |  120 +
 .../resources/schema/facts/testfact2_raw.xml|   75 +
 .../schema/facts/testfact2_raw_base.xml |   46 +
 .../resources/schema/facts/testfact3_base.xml   |  117 +
 .../schema/facts/testfact3_raw_base.xml |   44 +
 .../schema/facts/testfact4_raw_base.xml |   45 +
 .../resources/schema/facts/testfact5_base.xml   |  128 +
 .../schema/facts/testfact5_raw_base.xml |   38 +
 .../resources/schema/facts/testfact6_base.xml   |  117 +
 .../schema/facts/testfact_continuous.xml  

[3/7] lens git commit: LENS-1400: Convert CubeTestSetup to setup using xml files instead of code

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimtables/usertable.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimtables/usertable.xml 
b/lens-cube/src/test/resources/schema/dimtables/usertable.xml
new file mode 100644
index 000..055a958
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimtables/usertable.xml
@@ -0,0 +1,52 @@
+
+
+  
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+  
+  
+
+  
+  C1
+  
+
+
+  
+  
+
+
+  
+
+  
+
+
+  
+HOURLY
+  
+  C2
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/dimtables/ziptable.xml
--
diff --git a/lens-cube/src/test/resources/schema/dimtables/ziptable.xml 
b/lens-cube/src/test/resources/schema/dimtables/ziptable.xml
new file mode 100644
index 000..094031e
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/dimtables/ziptable.xml
@@ -0,0 +1,35 @@
+
+
+  
+
+
+
+  
+  
+
+
+
+
+
+  
+  
+
+  
+HOURLY
+  
+  C1
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/cheapfact.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/cheapfact.xml 
b/lens-cube/src/test/resources/schema/facts/cheapfact.xml
new file mode 100644
index 000..8a8d371
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/cheapfact.xml
@@ -0,0 +1,81 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+  
+  
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C99
+  
+
+  
+  
+
+
+  
+
+
+  
+
+ttd
+ttd2
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+MONTHLY
+QUARTERLY
+YEARLY
+  
+  C0
+  
+
+  
+
+
+  
+  
+
+
+  
+
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/summary1.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/summary1.xml 
b/lens-cube/src/test/resources/schema/facts/summary1.xml
new file mode 100644
index 000..199b991
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/summary1.xml
@@ -0,0 +1,81 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+
+  
+  
+
+  
+MINUTELY
+HOURLY
+DAILY
+  
+  C1
+  
+
+  
+
+
+  
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+  
+  C2
+  
+
+  
+  
+  
+
+
+  
+
+
+  
+
+pt
+it
+et
+  
+
+  
+
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lens/blob/112af59c/lens-cube/src/test/resources/schema/facts/summary2.xml
--
diff --git a/lens-cube/src/test/resources/schema/facts/summary2.xml 
b/lens-cube/src/test/resources/schema/facts/summary2.xml
new file mode 100644
index 000..c30ed75
--- /dev/null
+++ b/lens-cube/src/test/resources/schema/facts/summary2.xml
@@ -0,0 +1,83 @@
+
+
+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+
+
+
+
+
+
+
+
+
+  
+  
+
+  
+MINUTELY
+HOURLY
+DAILY
+  
+  C1
+  
+
+  
+
+
+  
+
+
+  
+
+dt
+  
+
+
+  
+MINUTELY
+HOURLY
+DAILY
+  
+  C2
+  
+
+  
+  
+  
+
+  

lens git commit: LENS-1398: Python client not raising exception in some cases

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master bd344c7ff -> a3045fee1


LENS-1398: Python client not raising exception in some cases


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a3045fee
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a3045fee
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a3045fee

Branch: refs/heads/master
Commit: a3045fee12b3e750b6c931f5221a480d415efc76
Parents: bd344c7
Author: Rajat Khandelwal 
Authored: Tue Mar 21 16:09:05 2017 +0530
Committer: Rajat Khandelwal 
Committed: Tue Mar 21 16:09:05 2017 +0530

--
 contrib/clients/python/lens/client/main.py |  4 +-
 contrib/clients/python/lens/client/query.py| 43 -
 contrib/clients/python/setup.py|  2 +
 contrib/clients/python/test/test_lensclient.py | 13 +--
 tools/conf/server/lens-site.xml|  6 +++
 5 files changed, 54 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/main.py
--
diff --git a/contrib/clients/python/lens/client/main.py 
b/contrib/clients/python/lens/client/main.py
index bf5d81e..1e9d88f 100644
--- a/contrib/clients/python/lens/client/main.py
+++ b/contrib/clients/python/lens/client/main.py
@@ -21,7 +21,8 @@ from .log import LensLogClient
 from .session import LensSessionClient
 from .query import LensQueryClient
 from .utils import xml_file_to_conf
-
+import logging
+logger = logging.getLogger(__name__)
 
 class LensClient(object):
 def __init__(self, base_url=None, username="", password="", database=None, 
conf=None):
@@ -35,6 +36,7 @@ class LensClient(object):
 self.base_url = base_url or conf.get('lens.server.base.url', 
"http://0.0.0.0:/lensapi;)
 if self.base_url[-1] != '/':
 self.base_url += "/"
+logger.debug("Using conf %s", conf)
 username = username or conf.get('lens.client.user.name', "anonymous")
 database = database or conf.get('lens.client.dbname')
 self.session = LensSessionClient(self.base_url, username, password, 
database, conf)

http://git-wip-us.apache.org/repos/asf/lens/blob/a3045fee/contrib/clients/python/lens/client/query.py
--
diff --git a/contrib/clients/python/lens/client/query.py 
b/contrib/clients/python/lens/client/query.py
index f82f0cb..df16cca 100644
--- a/contrib/clients/python/lens/client/query.py
+++ b/contrib/clients/python/lens/client/query.py
@@ -15,21 +15,26 @@
 # limitations under the License.
 #
 import codecs
+import csv
+import logging
 import time
 import zipfile
 
 import requests
-from six import string_types, BytesIO, StringIO, PY2, PY3
+from requests.exceptions import HTTPError
+from six import string_types, BytesIO, PY2, PY3
+
 from .models import WrappedJson
 from .utils import conf_to_xml
-import csv
 
+logger = logging.getLogger(__name__)
 long_type = int
 
 if PY3:
 from collections.abc import Iterable as Iterable
 elif PY2:
 from collections import Iterable as Iterable
+
 long_type = long
 
 
@@ -70,6 +75,7 @@ type_mappings = {'BOOLEAN': bool,
  }
 default_mapping = lambda x: x
 
+
 class LensQueryResult(Iterable):
 def __init__(self, custom_mappings=None):
 if custom_mappings is None:
@@ -93,6 +99,7 @@ class LensInMemoryResult(LensQueryResult):
 for row in self.rows:
 yield list(self._mapping(value.type)(value.value) if value else 
None for value in row['values'])
 
+
 class LensPersistentResult(LensQueryResult):
 def __init__(self, header, response, encoding=None, 
is_header_present=True, delimiter=",",
  custom_mappings=None):
@@ -107,6 +114,11 @@ class LensPersistentResult(LensQueryResult):
 def _parse_line(self, line):
 return 
list(self._mapping(self.header.columns[index].type)(line[index]) for index in 
range(len(line)))
 
+def get_csv_reader(self, file):
+if PY3:
+file = codecs.iterdecode(file, 'utf-8')
+return csv.reader(file, delimiter=self.delimiter)
+
 def __iter__(self):
 if self.is_zipped:
 byte_stream = BytesIO(self.response.content)
@@ -114,7 +126,7 @@ class LensPersistentResult(LensQueryResult):
 for name in self.zipfile.namelist():
 with self.zipfile.open(name) as single_file:
 if name[-3:] == 'csv':
-reader = csv.reader(single_file, 
delimiter=self.delimiter)
+reader = self.get_csv_reader(single_file)
 else:
 reader = single_file

[4/8] lens git commit: LENS-1389: Back Merge with master and fix lens-cube tests

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/2aaf6e0a/lens-regression/src/test/java/org/apache/lens/regression/client/ITKillQueryTests.java
--
diff --git 
a/lens-regression/src/test/java/org/apache/lens/regression/client/ITKillQueryTests.java
 
b/lens-regression/src/test/java/org/apache/lens/regression/client/ITKillQueryTests.java
new file mode 100644
index 000..ecc0c6b
--- /dev/null
+++ 
b/lens-regression/src/test/java/org/apache/lens/regression/client/ITKillQueryTests.java
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.client;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import javax.ws.rs.client.WebTarget;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.query.*;
+import org.apache.lens.regression.core.constants.QueryInventory;
+import org.apache.lens.regression.core.helpers.*;
+import org.apache.lens.regression.core.testHelper.BaseTestClass;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import org.testng.Assert;
+import org.testng.annotations.*;
+
+
+public class ITKillQueryTests extends BaseTestClass {
+
+  WebTarget servLens;
+  private String sessionHandleString;
+
+  private final String hdfsJarPath = lens.getServerHdfsUrl() + "/tmp";
+  private final String localJarPath = new File("").getAbsolutePath() + 
"/lens-regression/target/testjars/";
+  private final String hiveUdfJar = "hiveudftest.jar";
+  private final String serverResourcePath = "/tmp/regression/resources";
+  String sleepQuery = QueryInventory.getSleepQuery("5");
+
+  private static Logger logger = Logger.getLogger(ITKillQueryTests.class);
+
+  @BeforeClass(alwaysRun = true)
+  public void initialize() throws IOException, JAXBException, LensException, 
IllegalAccessException,
+  InstantiationException {
+servLens = ServiceManagerHelper.init();
+logger.info("Creating a new Session");
+sessionHandleString = sHelper.openSession(lens.getCurrentDB());
+
+//TODO : Enable when udf registration per driver is fixed
+/*  HadoopUtil.uploadJars(localJarPath + "/" + hiveUdfJar, hdfsJarPath);
+logger.info("Adding jar for making query to run for longer period of 
time");
+sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar);
+QueryHandle queryHandle = (QueryHandle) 
qHelper.executeQuery(QueryInventory.SLEEP_FUNCTION).getData();*/
+  }
+
+  @BeforeMethod(alwaysRun = true)
+  public void setUp(Method method) throws Exception {
+logger.info("Test Name: " + method.getName());
+  }
+
+
+  @AfterClass(alwaysRun = true)
+  public void closeSession() throws Exception {
+logger.info("Closing Session");
+sHelper.closeSession();
+  }
+
+
+  @Test(enabled = true)
+  public void killQueryByHandle() throws Exception {
+
+QueryHandle qH = (QueryHandle) qHelper.executeQuery(sleepQuery).getData();
+logger.info("QUERY HANDLE : " + qH);
+
+QueryStatus queryStatus = qHelper.waitForQueryToRun(qH);
+Assert.assertEquals(queryStatus.getStatus(), QueryStatus.Status.RUNNING, 
"Query is Not Running");
+
+qHelper.killQueryByQueryHandle(qH);
+
+queryStatus = qHelper.getQueryStatus(qH);
+Assert.assertEquals(queryStatus.getStatus(), QueryStatus.Status.CANCELED, 
"Query is Not Running");
+  }
+
+
+  @Test(enabled = true)
+  public void killQueryByUser() throws Exception {
+
+String diffUser = "diff";
+String diffPass = "diff";
+
+String newSessionHandleSring = sHelper.openSession(diffUser, diffPass, 
lens.getCurrentDB());
+
+logger.info("Adding jar for making query to run for longer period of 
time");
+sHelper.addResourcesJar(hdfsJarPath + "/" + hiveUdfJar, 
newSessionHandleSring);
+
+QueryHandle queryHandle1 = (QueryHandle) 
qHelper.executeQuery(sleepQuery).getData();
+logger.info("1st QUERY HANDLE : " + queryHandle1);
+
+QueryHandle queryHandle2 = (QueryHandle) 
qHelper.executeQuery(sleepQuery).getData();
+logger.info("2nd QUERY HANDLE : " + queryHandle2);
+
+QueryHandle queryHandle3 = (QueryHandle) qHelper.executeQuery(sleepQuery, 
null,
+

lens git commit: LENS-1369: Fixing Druid query rewriter issues as per the current state of plyql Fixes for the following issues : 1. Plyql BETWEEN clause excludes both ends. Feature addition to specif

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 4ae48c749 -> 08d79631b


LENS-1369: Fixing Druid query rewriter issues as per the current state of plyql
Fixes for the following issues :
1. Plyql BETWEEN clause excludes both ends. Feature addition to specify range 
boundary type
2. Plyql doesn't support having/orderby queries properly. Code to throw 
exception in those cases.


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08d79631
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08d79631
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08d79631

Branch: refs/heads/master
Commit: 08d79631bbe801a768369ab45f662b2785927110
Parents: 4ae48c7
Author: Rajitha R 
Authored: Wed Nov 9 19:06:57 2016 +0530
Committer: Rajat Khandelwal 
Committed: Wed Nov 9 19:06:57 2016 +0530

--
 .../lens/cube/metadata/FactPartition.java   | 10 ++
 .../lens/cube/parse/BetweenTimeRangeWriter.java | 16 
 .../lens/cube/parse/CubeQueryConfUtil.java  |  4 +
 .../cube/parse/TestBetweenTimeRangeWriter.java  | 94 ++-
 .../lens/cube/parse/TestTimeRangeWriter.java|  9 ++
 .../lens/driver/jdbc/DruidSQLRewriter.java  | 85 -
 .../org/apache/lens/driver/jdbc/JDBCDriver.java |  5 +-
 .../driver/jdbc/JDBCDriverConfConstants.java| 18 
 .../lens/driver/jdbc/TestDruidSQLRewriter.java  | 99 +---
 .../apache/lens/driver/jdbc/TestJdbcDriver.java | 22 +
 .../drivers/jdbc/druid/jdbcdriver-site.xml  |  8 --
 src/site/apt/user/cli.apt   | 86 -
 12 files changed, 362 insertions(+), 94 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/08d79631/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java 
b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
index 355a1f0..1694b80 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
@@ -86,6 +86,16 @@ public class FactPartition implements 
Comparable {
 }
   }
 
+  public FactPartition previous() throws LensException {
+return new FactPartition(getPartCol(), getTimePartition().previous(), 
getContainingPart(), getStorageTables());
+
+  }
+
+  public FactPartition next() throws LensException {
+return new FactPartition(getPartCol(), getTimePartition().next(), 
getContainingPart(), getStorageTables());
+
+  }
+
   public String getPartString() {
 return period.format(partSpec);
   }

http://git-wip-us.apache.org/repos/asf/lens/blob/08d79631/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
--
diff --git 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
index 046149b..c8b8129 100644
--- 
a/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
+++ 
b/lens-cube/src/main/java/org/apache/lens/cube/parse/BetweenTimeRangeWriter.java
@@ -25,6 +25,8 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.FactPartition;
 import org.apache.lens.server.api.error.LensException;
 
+import com.google.common.collect.BoundType;
+
 
 /**
  * Writes partitions queried in timerange as between clause.
@@ -41,6 +43,12 @@ public class BetweenTimeRangeWriter implements 
TimeRangeWriter {
 boolean useBetweenOnly = 
cubeQueryContext.getConf().getBoolean(CubeQueryConfUtil.BETWEEN_ONLY_TIME_RANGE_WRITER,
   CubeQueryConfUtil.DEFAULT_BETWEEN_ONLY_TIME_RANGE_WRITER);
 
+//Fetch the date start and end bounds from config
+BoundType startBound = 
BoundType.valueOf(cubeQueryContext.getConf().get(CubeQueryConfUtil.START_DATE_BOUND_TYPE,
+  CubeQueryConfUtil.DEFAULT_START_BOUND_TYPE));
+BoundType endBound = 
BoundType.valueOf(cubeQueryContext.getConf().get(CubeQueryConfUtil.END_DATE_BOUND_TYPE,
+  CubeQueryConfUtil.DEFAULT_END_BOUND_TYPE));
+
 StringBuilder partStr = new StringBuilder();
 if (!useBetweenOnly && rangeParts.size() == 1) {
   partStr.append("(");
@@ -75,6 +83,14 @@ public class BetweenTimeRangeWriter implements 
TimeRangeWriter {
   FactPartition start = parts.first();
   FactPartition end = parts.last();
 
+  if (startBound.equals(BoundType.OPEN)) {
+start = start.previous();
+  }
+
+  if (endBound.equals(BoundType.OPEN)) {
+end = end.next();
+  }
+
   String partCol = 

lens git commit: LENS-1358: Lens server snapshotter failing when there's a jdbc query running while snapshotting

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 38ab6c608 -> 34f15c6e2


LENS-1358: Lens server snapshotter failing when there's a jdbc query running 
while snapshotting


Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/34f15c6e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/34f15c6e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/34f15c6e

Branch: refs/heads/master
Commit: 34f15c6e24ef4604ef431765c3f23c799a44bec1
Parents: 38ab6c6
Author: Rajat Khandelwal 
Authored: Mon Oct 17 12:58:48 2016 +0530
Committer: Rajat Khandelwal 
Committed: Mon Oct 17 12:58:48 2016 +0530

--
 .../apache/lens/server/api/query/QueryContext.java  |  4 ++--
 .../server/api/query/TestAbstractQueryContext.java  | 16 ++--
 .../lens/server/api/query/TestQueryContext.java |  2 +-
 3 files changed, 9 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/lens/blob/34f15c6e/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 63d3539..64c7145 100644
--- 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -201,8 +201,7 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   @Getter
   @Setter
   private transient Future queryLauncher;
-
-  private final List 
driverStatusUpdateListeners = Lists.newArrayList();
+  transient List driverStatusUpdateListeners 
= Lists.newArrayList();
   @Getter
   @Setter
   List failedAttempts = Lists.newArrayList();
@@ -310,6 +309,7 @@ public class QueryContext extends AbstractQueryContext 
implements FailureContext
   public void initTransientState() {
 super.initTransientState();
 statusUpdateFailures = new StatusUpdateFailureContext();
+driverStatusUpdateListeners = Lists.newArrayList();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/lens/blob/34f15c6e/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
--
diff --git 
a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
 
b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
index 36cc8fa..f5af0d1 100644
--- 
a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
+++ 
b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
@@ -73,26 +73,22 @@ public class TestAbstractQueryContext {
   public void testTransientState() throws LensException, IOException, 
ClassNotFoundException {
 MockQueryContext ctx = new MockQueryContext();
 ByteArrayOutputStream bios = new ByteArrayOutputStream();
-ObjectOutputStream out = new ObjectOutputStream(bios);
-byte[] ctxBytes = null;
-try {
+byte[] ctxBytes;
+try (ObjectOutputStream out = new ObjectOutputStream(bios)) {
   out.writeObject(ctx);
   ctxBytes = bios.toByteArray();
-} finally {
-  out.close();
 }
 ByteArrayInputStream bais = new ByteArrayInputStream(ctxBytes);
-ObjectInputStream in = new ObjectInputStream(bais);
-MockQueryContext ctxRead = null;
-try {
+MockQueryContext ctxRead;
+try (ObjectInputStream in = new ObjectInputStream(bais)) {
   ctxRead = (MockQueryContext) in.readObject();
-} finally {
-  in.close();
 }
 ctxRead.initTransientState();
 ctxRead.setConf(ctx.getConf());
 assertNotNull(ctxRead.getHiveConf());
 assertNotNull(ctxRead.statusUpdateFailures);
+assertNotNull(ctxRead.driverStatusUpdateListeners);
+assertEquals(ctxRead.driverStatusUpdateListeners.size(), 0);
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/lens/blob/34f15c6e/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
--
diff --git 
a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
 
b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
index 1560bf1..d1b3d5f 100644
--- 
a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
+++ 
b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
@@ -74,7 +74,7 @@ public class TestQueryContext {
   }
 
   @Test
-  public void 

[2/4] lens git commit: LENS-743: Query retry framework for retrying upon transient failures

[prongs]

http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
new file mode 100644
index 000..6169744
--- /dev/null
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * The Class StatusChange.
+ */
+public abstract class StatusChange extends QueryEvent {
+
+  /**
+   * Instantiates a new status change.
+   *
+   * @param eventTime the event time
+   * @param prev  the prev
+   * @param current   the current
+   * @param handlethe handle
+   */
+  public StatusChange(long eventTime, QueryStatus.Status prev, 
QueryStatus.Status current, QueryHandle handle) {
+super(eventTime, prev, current, handle);
+  }
+
+  /**
+   * Check current state.
+   *
+   * @param status the status
+   */
+  protected void checkCurrentState(QueryStatus.Status status) {
+if (currentValue != status) {
+  throw new IllegalStateException("Invalid query state: " + currentValue + 
" query:" + queryHandle.toString());
+}
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
--
diff --git 
a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
 
b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
new file mode 100644
index 000..5ea5710
--- /dev/null
+++ 
b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.retry;
+
+import java.io.Serializable;
+
+/**
+ * A backoff retry handler.
+ *
+ * This allows a backoff on any call, so provides methods whether we can try 
the operation now,
+ * whats next time when operation can be performed and whether operation has 
exhausted all retries.
+ * Callers of this can do the following :
+ *
+ *  if (handler.canTryOpNow(FailureContext)) {
+ *try {
+ *  tryCallerOperation();
+ *  FailureContext.clear();
+ *} catch (any Transient Exception) {
+ *  FailureContext.updateFailure();
+ *  if (!handler.hasExhaustedRetries(FailureContext)) {
+ *// will be tried later again
+ *  }
+ *  throw exception;
+ *}
+ *  }
+ *
+ *  Note that this is only one of the possible use cases, other complex use 
cases are in retry framework.
+ */
+public interface BackOffRetryHandler extends 
Serializable {
+
+  /**
+   * To know whether operation can be done now.
+   *
+   * @param failContext FailureContext holding failures till now.
+   *
+   * @return true if operation can be done now, false otherwise.
+   */
+  boolean canTryOpNow(FC failContext);
+
+  /**
+   * Get the time when the operation can be done next.
+   *
+   * @param failContext FC 

[1/4] lens git commit: LENS-743: Query retry framework for retrying upon transient failures

[prongs]

Repository: lens
Updated Branches:
  refs/heads/master 182f6dcac -> 38ab6c608


http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
--
diff --git 
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
 
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index c683a2c..00130d0 100644
--- 
a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++ 
b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -41,8 +41,8 @@ import org.apache.lens.server.api.LensErrorInfo;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.events.SchedulerAlarmEvent;
 import org.apache.lens.server.api.health.HealthStatus;
-import org.apache.lens.server.api.query.QueryEnded;
 import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.events.QueryEnded;
 import org.apache.lens.server.api.scheduler.SchedulerService;
 import org.apache.lens.server.error.LensSchedulerErrorCode;
 import org.apache.lens.server.session.LensSessionImpl;

http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
 
b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
index 46adb7b..20243f4 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
@@ -27,16 +27,26 @@ import java.util.Comparator;
 
 import org.apache.lens.api.Priority;
 import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.comparators.ChainedComparator;
+import org.apache.lens.server.api.query.comparators.FIFOQueryComparator;
+import org.apache.lens.server.api.query.comparators.QueryCostComparator;
+import org.apache.lens.server.api.query.comparators.QueryPriorityComparator;
 import org.apache.lens.server.api.query.cost.QueryCost;
 
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.google.common.collect.Lists;
+
 public class QueryContextComparatorTest {
 
   private final Comparator priorityComparator = new 
QueryPriorityComparator();
   private final Comparator costComparator = new 
QueryCostComparator();
-
+  private final Comparator fifoComparator = new 
FIFOQueryComparator();
+  private final Comparator priorityAndFifoComparator
+= new ChainedComparator<>(Lists.newArrayList(priorityComparator, 
fifoComparator));
+  private final Comparator costAndFifoComparator
+= new ChainedComparator<>(Lists.newArrayList(costComparator, 
fifoComparator));
 
 
   @DataProvider
@@ -61,7 +71,7 @@ public class QueryContextComparatorTest {
 when(query2.getSelectedDriverQueryCost()).thenReturn(qcO2);
 
 when(qcO1.compareTo(qcO2)).thenReturn(resultOfQueryCostCompare);
-assertEquals(costComparator.compare(query1, query2), expectedResult);
+assertEquals(costAndFifoComparator.compare(query1, query2), 
expectedResult);
   }
 
   @Test
@@ -73,7 +83,7 @@ public class QueryContextComparatorTest {
 QueryContext query2 = mock(QueryContext.class);
 when(query2.getPriority()).thenReturn(Priority.LOW); // Ordinal = 3
 
-assertEquals(priorityComparator.compare(query1, query2), -2);
+assertEquals(priorityAndFifoComparator.compare(query1, query2), -2);
   }
 
 
@@ -114,8 +124,8 @@ public class QueryContextComparatorTest {
 
 // Cost and Priority both are same, hence the comparison should happen
 // on query submission time
-assertEquals(priorityComparator.compare(query1, query2), expectedResult);
-assertEquals(costComparator.compare(query1, query2), expectedResult);
+assertEquals(priorityAndFifoComparator.compare(query1, query2), 
expectedResult);
+assertEquals(costAndFifoComparator.compare(query1, query2), 
expectedResult);
 
   }
 }

http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
--
diff --git 
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java 
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index 526accc..b906776 100644
--- 
a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++ 
b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -39,7 +39,7 @@ import org.apache.lens.server.api.events.AsyncEventListener;