LENS-1420 : Bug fixes in cube segmentation final rewritten query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7c1171c8 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7c1171c8 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7c1171c8 Branch: refs/heads/current-release-line Commit: 7c1171c8dae842233d0db85bc15c00fcd37aa3de Parents: 139f029 Author: Sushil Mohanty <[email protected]> Authored: Mon May 15 13:27:54 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Mon May 15 13:27:54 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/ExpressionResolver.java | 4 +++- .../lens/cube/parse/UnionQueryWriter.java | 25 ++++++++++++++++---- .../parse/TestCubeSegmentationRewriter.java | 22 ++++++++++++++++- .../cube/parse/TestUnionAndJoinCandidates.java | 13 ++++++++++ .../resources/schema/cubes/base/b1c1cube.xml | 3 +++ .../test/resources/schema/cubes/base/b1cube.xml | 3 +++ .../resources/schema/cubes/base/b2c1cube.xml | 3 +++ .../test/resources/schema/cubes/base/b2cube.xml | 3 +++ .../resources/schema/cubes/base/basecube.xml | 7 ++++++ .../resources/schema/cubes/base/testcube.xml | 3 +++ .../cubes/derived/union_join_ctx_der1.xml | 1 + 11 files changed, 80 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java index f86a84a..7cad400 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java @@ -426,14 +426,16 @@ class ExpressionResolver implements ContextRewriter { // Having ast is not copied now, it's maintained in cubeQueryContext, each fact processes that serially. if (queryAST.getHavingAST() != null) { replaceAST(cubeql, queryAST.getHavingAST()); - } else { + } else if (cubeql.getHavingAST() != null) { replaceAST(cubeql, cubeql.getHavingAST()); + queryAST.setHavingAST(MetastoreUtil.copyAST(cubeql.getHavingAST())); } replaceAST(cubeql, queryAST.getOrderByAST()); } private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws LensException { if (node == null) { + return; } // Traverse the tree and resolve expression columns http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index bd7134b..6c0d91a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -93,6 +93,9 @@ public class UnionQueryWriter extends SimpleHQLContext { for (StorageCandidateHQLContext sc : storageCandidates) { storageCandidateToSelectAstMap.put(sc.getStorageCandidate().toString(), new ASTNode(new CommonToken(TOK_SELECT, "TOK_SELECT"))); + if (sc.getQueryAst().getHavingAST() != null) { + cubeql.setHavingAST(sc.getQueryAst().getHavingAST()); + } sc.getQueryAst().setHavingAST(null); sc.getQueryAst().setOrderByAST(null); sc.getQueryAst().setLimitValue(null); @@ -196,7 +199,11 @@ public class UnionQueryWriter extends SimpleHQLContext { ASTNode outerOrderby = new ASTNode(child); ASTNode tokNullsChild = (ASTNode) child.getChild(0); ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); + if (((ASTNode) tokNullsChild.getChild(0)).getToken().getType() == HiveParser.DOT) { + outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); + } else { + outerTokNullsChild.addChild(tokNullsChild); + } outerOrderby.addChild(outerTokNullsChild); outerExpression.addChild(outerOrderby); } @@ -571,8 +578,12 @@ public class UnionQueryWriter extends SimpleHQLContext { return outerAST; } else { ASTNode outerAST = getDotAST(cubeql.getCube().getName(), alias); - (isSelectAst ? innerToOuterSelectASTs : innerToOuterHavingASTs) - .put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + HashableASTNode innerAST = new HashableASTNode(innerSelectASTWithoutAlias); + if (isSelectAst && !innerToOuterSelectASTs.containsKey(innerAST)) { + innerToOuterSelectASTs.put(innerAST, outerAST); + } else if (!isSelectAst && !innerToOuterHavingASTs.containsKey(innerAST)) { + innerToOuterHavingASTs.put(innerAST, outerAST); + } return outerAST; } } @@ -593,8 +604,12 @@ public class UnionQueryWriter extends SimpleHQLContext { //TODO: take care or non-transitive aggregate functions outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText()))); outerAST.addChild(dotAST); - (isSelectAst ? innerToOuterSelectASTs : innerToOuterHavingASTs) - .put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST); + HashableASTNode innerAST = new HashableASTNode(innerSelectASTWithoutAlias); + if (isSelectAst && !innerToOuterSelectASTs.containsKey(innerAST)) { + innerToOuterSelectASTs.put(innerAST, outerAST); + } else if (!isSelectAst && !innerToOuterHavingASTs.containsKey(innerAST)) { + innerToOuterHavingASTs.put(innerAST, outerAST); + } return outerAST; } http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java index 7f8662a..fe52600 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeSegmentationRewriter.java @@ -41,6 +41,7 @@ import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries; import static org.apache.commons.lang3.time.DateUtils.addDays; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import static com.google.common.collect.Lists.newArrayList; @@ -203,7 +204,7 @@ public class TestCubeSegmentationRewriter extends TestQueryRewrite { } @Test - public void testQueryWithWhereHavingGroupby() throws Exception { + public void testQueryWithWhereHavingGroupbyOrderby() throws Exception { String userQuery = "select cityid, msr2, segmsr1 from testcube where cityname='blah' and " + TWO_DAYS_RANGE + " group by cityid having segmsr1 > 1 and msr2 > 2"; CubeQueryContext ctx = rewriteCtx(userQuery, getConf()); @@ -231,6 +232,25 @@ public class TestCubeSegmentationRewriter extends TestQueryRewrite { "select testcube.alias0 as cityid, sum(testcube.alias1) as msr2, sum(testcube.alias2) as segmsr1 from ( ", ") as testcube group by testcube.alias0 having ((sum((testcube.alias2)) > 1) and (sum((testcube.alias1)) > 2)", newArrayList(query1, query2, query3)); + + // Expression in having + userQuery = "select cityid, segmsr1 from testcube where cityname='blah' and " + + TWO_DAYS_RANGE + " having citysegmsr1 > 20"; + String rewrittenQuery = rewrite(userQuery, getConf()); + assertTrue(rewrittenQuery.toLowerCase().endsWith("sum(case when ((cubecity.name) = 'foo') " + + "then (testcube.segmsr1) end) > 20)")); + + // Order by on alias + userQuery = "select cityid as `city_id_alias`, segmsr1 from testcube where cityname='blah' and " + + TWO_DAYS_RANGE + " order by city_id_alias"; + rewrittenQuery = rewrite(userQuery, getConf()); + assertTrue(rewrittenQuery.toLowerCase().endsWith("order by city_id_alias asc")); + + // Order by on column but the final query rewritten with alias + userQuery = "select cityid as `city_id_alias`, segmsr1 from testcube where cityname='blah' and " + + TWO_DAYS_RANGE + " order by cityid"; + rewrittenQuery = rewrite(userQuery, getConf()); + assertTrue(rewrittenQuery.toLowerCase().endsWith("order by city_id_alias asc")); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java index 0119f2e..9422a5c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java @@ -77,6 +77,19 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite { } @Test + public void testMultipleDimAttributeReferingSameJoinChain() throws ParseException, LensException { + String colsSelected = " union_join_ctx_cityid, union_join_ctx_cityname, union_join_ctx_dup_cityname, " + + " sum(union_join_ctx_msr1) "; + String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); + assertTrue(rewrittenQuery.contains("UNION ALL")); + // union_join_ctx_cityname and union_join_ctx_dup_cityname are refering to same join chain + // in the final query they both will have the same alias repeated twice in each union query. + assertEquals(StringUtil.count(rewrittenQuery, "basecube.alias1"), 4); + } + + @Test public void testFinalCandidateRewrittenQuery() throws ParseException, LensException { try { // Query with non projected measure in having clause. http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/b1c1cube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1c1cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b1c1cube.xml index 6ab528a..69b3ab7 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b1c1cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b1c1cube.xml @@ -171,6 +171,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml index ef0e33d..e3e9909 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b1cube.xml @@ -177,6 +177,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/b2c1cube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/b2c1cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b2c1cube.xml index c71540b..1ea017e 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b2c1cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b2c1cube.xml @@ -171,6 +171,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml index f0a80be..e5cf2c0 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/b2cube.xml @@ -177,6 +177,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/basecube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml index b8585ac..55099c8 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/basecube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/basecube.xml @@ -94,6 +94,10 @@ description="union_join_ctx_city name"> <chain_ref_column chain_name="cubecityjoinunionctx" ref_col="name" dest_table="citydim"/> </dim_attribute> + <dim_attribute _type="string" name="union_join_ctx_dup_cityname" display_string="union_join_ctx_city name" + description="union_join_ctx_city name"> + <chain_ref_column chain_name="cubecityjoinunionctx" ref_col="name" dest_table="citydim"/> + </dim_attribute> <dim_attribute _type="string" name="cityname" display_string="city name" description="city name"> <chain_ref_column chain_name="cubecity" ref_col="name" dest_table="citydim"/> </dim_attribute> @@ -228,6 +232,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/base/testcube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml index 2a11c88..f123341 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml @@ -164,6 +164,9 @@ <expression _type="string" name="singlecolchainfield" display_string="cubecityname" description="cubecity.name"> <expr_spec expr="cubecity.name"/> </expression> + <expression _type="string" name="citysegmsr1" display_string="city segmsr1" description="city segmsr1 "> + <expr_spec expr="sum(case when cubecity.name = 'foo' then segmsr1 end)"/> + </expression> <expression _type="double" name="msr8" display_string="Sixth Msr" description="measure expression"> <expr_spec expr="msr2 + msr3"/> </expression> http://git-wip-us.apache.org/repos/asf/lens/blob/7c1171c8/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml index 1308483..69eda6d 100644 --- a/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml +++ b/lens-cube/src/test/resources/schema/cubes/derived/union_join_ctx_der1.xml @@ -40,6 +40,7 @@ <dim_attr_names> <attr_name>union_join_ctx_zipcode</attr_name> <attr_name>union_join_ctx_cityname</attr_name> + <attr_name>union_join_ctx_dup_cityname</attr_name> <attr_name>d_time</attr_name> <attr_name>union_join_ctx_cityid</attr_name> </dim_attr_names>
