Repository: lens Updated Branches: refs/heads/lens-1381 363f132d1 -> d45c5384c
LENS-1403 : Measures getting repeated in inner select of union query Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d45c5384 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d45c5384 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d45c5384 Branch: refs/heads/lens-1381 Commit: d45c5384ccab119aa263e3bc4b2c3a6c78f8c993 Parents: 363f132 Author: Sushil Mohanty <[email protected]> Authored: Wed Mar 29 15:24:04 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Wed Mar 29 15:24:04 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/StorageCandidate.java | 2 + .../lens/cube/parse/UnionQueryWriter.java | 20 +++++--- .../lens/cube/parse/TestBaseCubeQueries.java | 30 +++++------ .../cube/parse/TestUnionAndJoinCandidates.java | 53 ++++++++++++-------- 4 files changed, 63 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 17f3af8..628e9aa 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -340,6 +340,8 @@ public class StorageCandidate implements Candidate, CandidateTable { if (this == cubeql.getPickedCandidate()) { CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql); updateOrderByWithFinalAlias(queryAst.getOrderByAST(), queryAst.getSelectAST()); + } else { + queryAst.setHavingAST(null); } return CandidateUtil .buildHQLString(queryAst.getSelectString(), fromString, whereString, queryAst.getGroupByString(), http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java index f9717fa..f2325f1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java @@ -142,7 +142,11 @@ public class UnionQueryWriter { ASTNode expr = innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(node)) ? innerToOuterSelectASTs.get(new HQLParser.HashableASTNode(node)) : innerToOuterHavingASTs.get(new HQLParser.HashableASTNode(node)); - node.getParent().setChild(0, expr); + if (node.getChildCount() > 1) { + node.replaceChildren(1, 1, expr.getChild(1)); + } else { + node.replaceChildren(0, 0, expr); + } } } for (int i = 0; i < node.getChildCount(); i++) { @@ -191,7 +195,7 @@ public class UnionQueryWriter { ASTNode outerOrderby = new ASTNode(child); ASTNode tokNullsChild = (ASTNode) child.getChild(0); ASTNode outerTokNullsChild = new ASTNode(tokNullsChild); - outerTokNullsChild.addChild(getOuterAST((ASTNode) tokNullsChild.getChild(0), null, aliasDecider, null, true, cubeql.getBaseCube().getDimAttributeNames())); + outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild))); outerOrderby.addChild(outerTokNullsChild); outerExpression.addChild(outerOrderby); } @@ -299,8 +303,7 @@ public class UnionQueryWriter { private List<ASTNode> getProjectedNonDefaultPhrases() { List<ASTNode> phrases = new ArrayList<>(); - int selectPhraseCount = cubeql.getSelectPhrases().size(); - for (int i = 0; i < selectPhraseCount; i++) { + for (int i = 0; i < storageCandidates.iterator().next().getQueryAst().getSelectAST().getChildCount(); i++) { for (StorageCandidate sc : storageCandidates) { ASTNode selectAST = sc.getQueryAst().getSelectAST(); if (isNodeDefault((ASTNode) selectAST.getChild(i))) { @@ -359,10 +362,13 @@ public class UnionQueryWriter { } } } - updateOuterSelectDuplicateAliases(queryAst.getSelectAST(), aliasMap); + updateOuterASTDuplicateAliases(queryAst.getSelectAST(), aliasMap); + if (queryAst.getHavingAST() != null) { + updateOuterASTDuplicateAliases(queryAst.getHavingAST(), aliasMap); + } } - public void updateOuterSelectDuplicateAliases(ASTNode node, + public void updateOuterASTDuplicateAliases(ASTNode node, Map<String, List<String>> aliasMap) { if (node.getToken().getType() == HiveParser.DOT) { String table = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).toString(); @@ -380,7 +386,7 @@ public class UnionQueryWriter { } for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); - updateOuterSelectDuplicateAliases(child, aliasMap); + updateOuterASTDuplicateAliases(child, aliasMap); } } http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java index 591b24b..ba8a5e4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java @@ -1024,11 +1024,11 @@ public class TestBaseCubeQueries extends TestQueryRewrite { + "having msr12 > 2 and roundedmsr2 > 0 and msr2 > 100", conf); expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " - + "sum(0.0) as `alias3`, sum(0.0) as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11", + + "sum(0.0) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, " - + "sum((basecube.msr2)) as `alias3`, sum((basecube.msr2)) as `alias4` FROM ", null, + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); @@ -1036,8 +1036,8 @@ public class TestBaseCubeQueries extends TestQueryRewrite { compareContains(expected2, hqlQuery); assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, " + "(basecube.alias1) as `dim11` from"), hqlQuery); - assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and " - + "(round((sum((basecube.alias4)) / 1000)) > 0) and (sum((basecube.alias4)) > 100))"), hqlQuery); + assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) " + + "and (round((sum((basecube.alias3)) / 1000)) > 0) and (sum((basecube.alias3)) > 100))"), hqlQuery); hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE + "having msr12+roundedmsr2 <= 1000", conf); @@ -1063,13 +1063,13 @@ public class TestBaseCubeQueries extends TestQueryRewrite { hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE + "having msr12 > 2 and roundedmsr2 > 0 and msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, - "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12))" - + " as `alias2`, sum(0.0) as `alias3`, sum((basecube.msr12)) as `alias4`, sum(0.0) as `alias5` FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " + + "sum(0.0) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, sum((basecube.msr2)) " - + "as `alias3`, sum(0.0) as `alias4`, sum((basecube.msr2)) as `alias5` FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); @@ -1078,20 +1078,20 @@ public class TestBaseCubeQueries extends TestQueryRewrite { assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) " + "as `dim11` from "), hqlQuery); assertTrue(hqlQuery.contains("UNION ALL") - && hqlQuery.endsWith("HAVING ((sum((basecube.alias4)) > 2) and (round((sum((basecube.alias5)) / 1000)) > 0) " - + "and ((sum((basecube.alias4)) + round((sum((basecube.alias5)) / 1000))) <= 1000))"), hqlQuery); + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and (round((sum((basecube.alias3)) / 1000)) > 0) " + + "and ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery); hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE + "having msr12 > 2 or roundedmsr2 > 0 or msr12+roundedmsr2 <= 1000", conf); expected1 = getExpectedQuery(cubeName, "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, " - + "sum(0.0) as `alias3`, sum((basecube.msr12)) as `alias4`, sum(0.0) as `alias5` FROM ", + + "sum(0.0) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE")); expected2 = getExpectedQuery(cubeName, - "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, sum((basecube.msr2)) " - + "as `alias3`, sum(0.0) as `alias4`, sum((basecube.msr2)) as `alias5` FROM ", + "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, " + + "sum((basecube.msr2)) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE")); @@ -1100,7 +1100,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite { assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) " + "as `dim11` from"), hqlQuery); assertTrue(hqlQuery.contains("UNION ALL") - && hqlQuery.endsWith("HAVING ((sum((basecube.alias4)) > 2) or (round((sum((basecube.alias5)) / 1000)) > 0) or " - + "((sum((basecube.alias4)) + round((sum((basecube.alias5)) / 1000))) <= 1000))"), hqlQuery); + && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) or (round((sum((basecube.alias3)) / 1000)) > 0) " + + "or ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery); } } http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java index 931f789..f5f7f3e 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java @@ -25,6 +25,8 @@ import static org.apache.lens.cube.parse.TestCubeRewriter.compareContains; import static org.testng.Assert.*; +import jodd.util.StringUtil; +import org.apache.hadoop.util.StringUtils; import org.apache.lens.server.api.LensServerAPITestUtil; import org.apache.lens.server.api.error.LensException; @@ -36,38 +38,47 @@ import org.testng.annotations.Test; public class TestUnionAndJoinCandidates extends TestQueryRewrite { - private Configuration testConf; + private Configuration conf; @BeforeTest public void setupDriver() throws Exception { - testConf = LensServerAPITestUtil.getConfiguration( - DISABLE_AUTO_JOINS, false, - ENABLE_SELECT_TO_GROUPBY, true, - ENABLE_GROUP_BY_TO_SELECT, true, - DISABLE_AGGREGATE_RESOLVER, false, - ENABLE_STORAGES_UNION, true); + conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), + //Supported storage + CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1", + // Storage tables + getValidStorageTablesKey("union_join_ctx_fact1"), "C1_union_join_ctx_fact1", + getValidStorageTablesKey("union_join_ctx_fact2"), "C1_union_join_ctx_fact2", + getValidStorageTablesKey("union_join_ctx_fact3"), "C1_union_join_ctx_fact3", + // Update periods + getValidUpdatePeriodsKey("union_join_ctx_fact1", "C1"), "DAILY", + getValidUpdatePeriodsKey("union_join_ctx_fact2", "C1"), "DAILY", + getValidUpdatePeriodsKey("union_join_ctx_fact3", "C1"), "DAILY"); + conf.setBoolean(DISABLE_AUTO_JOINS, false); + conf.setBoolean(ENABLE_SELECT_TO_GROUPBY, true); + conf.setBoolean(ENABLE_GROUP_BY_TO_SELECT, true); + conf.setBoolean(DISABLE_AGGREGATE_RESOLVER, false); + conf.setBoolean(ENABLE_STORAGES_UNION, true); } @Override public Configuration getConf() { - return new Configuration(testConf); + return new Configuration(); } @Test + public void testDuplicateProjectedFieldExclusion() throws ParseException, LensException { + String colsSelected = " union_join_ctx_cityid , union_join_ctx_msr1_greater_than_100, " + + " sum(union_join_ctx_msr1) "; + String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and " + + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")"; + String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf); + assertTrue(rewrittenQuery.contains("UNION ALL")); + assertEquals(StringUtil.count(rewrittenQuery, "sum((basecube.union_join_ctx_msr1))"), 2); + } + + @Test public void testFinalCandidateRewrittenQuery() throws ParseException, LensException { try { - Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(), - //Supported storage - CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1", - // Storage tables - getValidStorageTablesKey("union_join_ctx_fact1"), "C1_union_join_ctx_fact1", - getValidStorageTablesKey("union_join_ctx_fact2"), "C1_union_join_ctx_fact2", - getValidStorageTablesKey("union_join_ctx_fact3"), "C1_union_join_ctx_fact3", - // Update periods - getValidUpdatePeriodsKey("union_join_ctx_fact1", "C1"), "DAILY", - getValidUpdatePeriodsKey("union_join_ctx_fact2", "C1"), "DAILY", - getValidUpdatePeriodsKey("union_join_ctx_fact3", "C1"), "DAILY"); - // Query with non projected measure in having clause. String colsSelected = "union_join_ctx_cityid, sum(union_join_ctx_msr2) "; String having = " having sum(union_join_ctx_msr1) > 100"; @@ -153,4 +164,6 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite { getStorageToUpdatePeriodMap().clear(); } } + + }
