Repository: lens Updated Branches: refs/heads/master 9ea1513b4 -> 017c40310
LENS-1264 : Fix aggregate resolution with aggregate around dim attributes Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/017c4031 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/017c4031 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/017c4031 Branch: refs/heads/master Commit: 017c40310e39af81bb75c6888a2494dd4bd4304a Parents: 9ea1513 Author: Sushil Mohanty <[email protected]> Authored: Mon Aug 8 10:01:16 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Mon Aug 8 10:01:16 2016 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/AggregateResolver.java | 24 +++++++++++++++++++- .../apache/lens/cube/parse/CubeTestSetup.java | 5 ++++ .../lens/cube/parse/TestCubeRewriter.java | 18 +++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/017c4031/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index ef2ca4e..292868a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -26,6 +26,7 @@ import java.util.Iterator; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.CubeMeasure; +import org.apache.lens.cube.metadata.ExprColumn; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.server.api.error.LensException; @@ -102,12 +103,33 @@ class AggregateResolver implements ContextRewriter { // Check if any measure/aggregate columns and distinct clause used in // select tree. If not, update selectAST token "SELECT" to "SELECT DISTINCT" if (!hasMeasures(cubeql, cubeql.getSelectAST()) && !isDistinctClauseUsed(cubeql.getSelectAST()) - && !HQLParser.hasAggregate(cubeql.getSelectAST())) { + && !HQLParser.hasAggregate(cubeql.getSelectAST()) + && !isAggregateDimExprUsedInSelect(cubeql, cubeql.getSelectAST())) { cubeql.getSelectAST().getToken().setType(HiveParser.TOK_SELECTDI); } } } + private boolean isAggregateDimExprUsedInSelect(CubeQueryContext cubeql, ASTNode selectAST) throws LensException { + for (int i = 0; i < selectAST.getChildCount(); i++) { + ASTNode child = (ASTNode) selectAST.getChild(i); + String expr = HQLParser.getString((ASTNode) child.getChild(0).getChild(1)); + if (cubeql.getQueriedExprs().contains(expr)) { + for (Iterator<ExpressionResolver.ExpressionContext> itrContext = + cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator(); itrContext.hasNext();) { + for (Iterator<ExprColumn.ExprSpec> itrCol = + itrContext.next().getExprCol().getExpressionSpecs().iterator(); itrCol.hasNext();) { + ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr()); + if (HQLParser.isAggregateAST(exprAST)) { + return true; + } + } + } + } + } + return false; + } + // We need to traverse the clause looking for eligible measures which can be // wrapped inside aggregates // We have to skip any columns that are already inside an aggregate UDAF http://git-wip-us.apache.org/repos/asf/lens/blob/017c4031/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 48652f2..2631f40 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -659,6 +659,11 @@ public class CubeTestSetup { "substr(dim2big1, 5)")); exprs.add(new ExprColumn(new FieldSchema("asciicity", "String", "ascii cityname"), "ascii cityname substr", "ascii(cityname)")); + exprs.add(new ExprColumn(new FieldSchema("countofdistinctcityid", "int", "Count of Distinct CityId"), + "Count of Distinct CityId Expr", "count(distinct(cityid))")); + exprs.add(new ExprColumn(new FieldSchema("notnullcityid", "int", "Not null cityid"), + "Not null cityid Expr", "case when cityid is null then 0 else cityid end")); + Map<String, String> cubeProperties = new HashMap<String, String>(); cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(TEST_CUBE_NAME), "d_time,pt,it,et,test_time_dim,test_time_dim2"); http://git-wip-us.apache.org/repos/asf/lens/blob/017c4031/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 4dfd754..ed54f0c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -617,6 +617,7 @@ public class TestCubeRewriter extends TestQueryRewrite { public void testCubeGroupbyQuery() throws Exception { Configuration conf = getConf(); conf.set(DRIVER_SUPPORTED_STORAGES, "C2"); + String hqlQuery = rewrite("select name, SUM(msr2) from" + " testCube join citydim on testCube.cityid = citydim.id where " + TWO_DAYS_RANGE, conf); @@ -694,6 +695,23 @@ public class TestCubeRewriter extends TestQueryRewrite { " group by testcube.zipcode order by rzc asc", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); compareQueries(hqlQuery, expected); + //Dim attribute with aggregate function + hqlQuery = + rewrite("select countofdistinctcityid, zipcode from" + " testCube where " + TWO_DAYS_RANGE, conf); + expected = + getExpectedQuery(TEST_CUBE_NAME, "select " + " count(distinct (testcube.cityid)), (testcube.zipcode) FROM ", + null, " group by (testcube.zipcode)", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + compareQueries(hqlQuery, expected); + + //Dim attribute with single row function + hqlQuery = + rewrite("select notnullcityid, zipcode from" + " testCube where " + TWO_DAYS_RANGE, conf); + expected = + getExpectedQuery(TEST_CUBE_NAME, "select " + " distinct case when (testcube.cityid) is null then 0 " + + "else (testcube.cityid) end, (testcube.zipcode) FROM ", null, + "", getWhereForDailyAndHourly2days(TEST_CUBE_NAME, "C2_testfact")); + compareQueries(hqlQuery, expected); + // rewrite with expressions conf.setBoolean(DISABLE_AUTO_JOINS, false); conf.set(DRIVER_SUPPORTED_STORAGES, "C1, C2");
