http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java index d9e442d..2cf92b9 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java @@ -21,6 +21,7 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.metadata.DateFactory.*; import static org.apache.lens.cube.parse.CubeTestSetup.*; +import static org.apache.lens.cube.parse.TestCubeRewriter.compareQueries; import static org.testng.Assert.*; @@ -28,7 +29,6 @@ import java.util.*; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.metadata.SchemaGraph.TableRelationship; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -45,7 +45,6 @@ import org.testng.annotations.Test; public class TestJoinResolver extends TestQueryRewrite { private static HiveConf hconf = new HiveConf(TestJoinResolver.class); - private CubeMetastoreClient metastore; @BeforeTest public void setupInstance() throws Exception { @@ -53,124 +52,25 @@ public class TestJoinResolver extends TestQueryRewrite { hconf.setBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, false); hconf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, true); hconf.setBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, true); + hconf.setBoolean(CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER, false); hconf.setBoolean(CubeQueryConfUtil.ENABLE_FLATTENING_FOR_BRIDGETABLES, true); - this.metastore = CubeMetastoreClient.getInstance(hconf); } @AfterTest public void closeInstance() throws Exception { } - // testBuildGraph - graph correctness - @Test - public void testBuildGraph() throws Exception { - SchemaGraph schemaGraph = metastore.getSchemaGraph(); - CubeInterface cube = metastore.getCube(CubeTestSetup.TEST_CUBE_NAME); - Map<AbstractCubeTable, Set<TableRelationship>> graph = schemaGraph.getCubeGraph(cube); - printGraph(graph); - Assert.assertNotNull(graph); - - // Let's do some lookups - Set<TableRelationship> dim4Edges = graph.get(metastore.getDimension("testdim4")); - Assert.assertNull(dim4Edges); - dim4Edges = graph.get(metastore.getDimension("testdim3")); - Assert.assertNotNull(dim4Edges); - Assert.assertEquals(1, dim4Edges.size()); - - List<TableRelationship> edges = new ArrayList<TableRelationship>(dim4Edges); - TableRelationship dim4edge = edges.get(0); - Assert.assertEquals("id", dim4edge.getToColumn()); - Assert.assertEquals(metastore.getDimension("testdim4"), dim4edge.getToTable()); - Assert.assertEquals("testdim4id", dim4edge.getFromColumn()); - Assert.assertEquals(metastore.getDimension("testdim3"), dim4edge.getFromTable()); - } - - private void searchPaths(AbstractCubeTable source, AbstractCubeTable target, SchemaGraph graph) { - SchemaGraph.GraphSearch search = new SchemaGraph.GraphSearch(source, target, graph); - List<SchemaGraph.JoinPath> joinPaths = search.findAllPathsToTarget(); - - System.out.println("@@ " + source + " ==> " + target + " paths ="); - int i = 0; - for (SchemaGraph.JoinPath jp : joinPaths) { - Assert.assertEquals(jp.getEdges().get(0).getToTable(), source); - Assert.assertEquals(jp.getEdges().get(jp.getEdges().size() - 1).getFromTable(), target); - Collections.reverse(jp.getEdges()); - System.out.println(++i + " " + jp.getEdges()); - } - } - - @Test - public void testFindChain() throws Exception { - SchemaGraph schemaGraph = metastore.getSchemaGraph(); - schemaGraph.print(); - - // Search For all cubes and all dims to make sure that search terminates - for (CubeInterface cube : metastore.getAllCubes()) { - for (Dimension dim : metastore.getAllDimensions()) { - searchPaths(dim, (AbstractCubeTable) cube, schemaGraph); - } - } - - for (Dimension dim : metastore.getAllDimensions()) { - for (Dimension otherDim : metastore.getAllDimensions()) { - if (otherDim != dim) { - searchPaths(dim, otherDim, schemaGraph); - } - } - } - - // Assert for testcube - CubeInterface testCube = metastore.getCube("testcube"); - Dimension zipDim = metastore.getDimension("zipdim"); - Dimension cityDim = metastore.getDimension("citydim"); - Dimension testDim2 = metastore.getDimension("testDim2"); - - SchemaGraph.GraphSearch search = new SchemaGraph.GraphSearch(zipDim, (AbstractCubeTable) testCube, schemaGraph); - - List<SchemaGraph.JoinPath> paths = search.findAllPathsToTarget(); - Assert.assertEquals(6, paths.size()); - validatePath(paths.get(0), zipDim, (AbstractCubeTable) testCube); - validatePath(paths.get(1), zipDim, cityDim, (AbstractCubeTable) testCube); - validatePath(paths.get(2), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube); - validatePath(paths.get(3), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube); - validatePath(paths.get(4), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube); - validatePath(paths.get(5), zipDim, cityDim, testDim2, (AbstractCubeTable) testCube); - } - - private void validatePath(SchemaGraph.JoinPath jp, AbstractCubeTable... tables) { - Assert.assertTrue(!jp.getEdges().isEmpty()); - Set<AbstractCubeTable> expected = new HashSet<AbstractCubeTable>(Arrays.asList(tables)); - Set<AbstractCubeTable> actual = new HashSet<AbstractCubeTable>(); - for (TableRelationship edge : jp.getEdges()) { - if (edge.getFromTable() != null) { - actual.add(edge.getFromTable()); - } - if (edge.getToTable() != null) { - actual.add(edge.getToTable()); - } - } - - Assert.assertEquals(expected, actual, - "Edges: " + jp.getEdges().toString() + " Expected Tables: " + Arrays.toString(tables) + " Actual Tables: " - + actual.toString()); - } - - private void printGraph(Map<AbstractCubeTable, Set<TableRelationship>> graph) { - System.out.println("--Graph-Nodes=" + graph.size()); - for (AbstractCubeTable tab : graph.keySet()) { - System.out.println(tab.getName() + "::" + graph.get(tab)); - } - } - private String getAutoResolvedFromString(CubeQueryContext query) throws LensException { return query.getHqlContext().getFrom(); } @Test public void testAutoJoinResolver() throws Exception { + HiveConf conf = new HiveConf(hconf); + conf.setBoolean(CubeQueryConfUtil.DISABLE_AGGREGATE_RESOLVER, true); // Test 1 Cube + dim - String query = "select citydim.name, testDim2.name, testDim4.name, msr2 from testCube where " + TWO_DAYS_RANGE; - CubeQueryRewriter driver = new CubeQueryRewriter(hconf, hconf); + String query = "select cubeCity.name, dim2chain.name, dim4chain.name, msr2 from testCube where " + TWO_DAYS_RANGE; + CubeQueryRewriter driver = new CubeQueryRewriter(conf, conf); CubeQueryContext rewrittenQuery = driver.rewrite(query); String hql = rewrittenQuery.toHQL(); System.out.println("testAutoJoinResolverauto join HQL:" + hql); @@ -178,13 +78,13 @@ public class TestJoinResolver extends TestQueryRewrite { List<String> expectedClauses = new ArrayList<String>(); expectedClauses.add(getDbName() + "c1_testfact2_raw testcube"); expectedClauses.add(getDbName() - + "c1_citytable citydim on testcube.cityid = citydim.id and (citydim.dt = 'latest')"); + + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')"); expectedClauses.add(getDbName() - + "c1_testdim2tbl testdim2 on testcube.dim2 = testdim2.id and (testdim2.dt = 'latest')"); + + "c1_testdim2tbl dim2chain on testcube.dim2 = dim2chain.id and (dim2chain.dt = 'latest')"); expectedClauses.add(getDbName() - + "c1_testdim3tbl testdim3 on testdim2.testdim3id = testdim3.id and (testdim3.dt = 'latest')"); + + "c1_testdim3tbl testdim3 on dim2chain.testdim3id = testdim3.id and (testdim3.dt = 'latest')"); expectedClauses.add(getDbName() - + "c1_testdim4tbl testdim4 on testdim3.testdim4id = testdim4.id and (testdim4.dt = 'latest')"); + + "c1_testdim4tbl dim4chain on testdim3.testdim4id = dim4chain.id and (dim4chain.dt = 'latest')"); List<String> actualClauses = new ArrayList<String>(); for (String clause : StringUtils.splitByWholeSeparator(getAutoResolvedFromString(rewrittenQuery), "join")) { @@ -199,7 +99,7 @@ public class TestJoinResolver extends TestQueryRewrite { // Test 2 Dim only query expectedClauses.clear(); actualClauses.clear(); - String dimOnlyQuery = "select testDim2.name, testDim4.name FROM testDim2 where " + TWO_DAYS_RANGE; + String dimOnlyQuery = "select testDim2.name, dim4chain.name FROM testDim2 where " + TWO_DAYS_RANGE; rewrittenQuery = driver.rewrite(dimOnlyQuery); hql = rewrittenQuery.toHQL(); System.out.println("testAutoJoinResolverauto join HQL:" + hql); @@ -208,7 +108,7 @@ public class TestJoinResolver extends TestQueryRewrite { expectedClauses.add(getDbName() + "c1_testdim3tbl testdim3 on testdim2.testdim3id = testdim3.id and (testdim3.dt = 'latest')"); expectedClauses.add(getDbName() - + "c1_testdim4tbl testdim4 on testdim3.testdim4id = testdim4.id and (testdim4.dt = 'latest')"); + + "c1_testdim4tbl dim4chain on testdim3.testdim4id = dim4chain.id and (dim4chain.dt = 'latest')"); for (String clause : StringUtils.splitByWholeSeparator(getAutoResolvedFromString(rewrittenQuery), "join")) { if (StringUtils.isNotBlank(clause)) { actualClauses.add(clause.trim()); @@ -224,28 +124,27 @@ public class TestJoinResolver extends TestQueryRewrite { } @Test - public void testPartialJoinResolver() throws Exception { + public void testJoinFilters() throws Exception { String query = - "SELECT citydim.name, testDim4.name, msr2 " - + "FROM testCube left outer join citydim ON citydim.name = 'FOOBAR'" - + " right outer join testDim4 on testDim4.name='TESTDIM4NAME'" + " WHERE " + TWO_DAYS_RANGE; - CubeQueryRewriter driver = new CubeQueryRewriter(hconf, hconf); - CubeQueryContext rewrittenQuery = driver.rewrite(query); - String hql = rewrittenQuery.toHQL(); - System.out.println("testPartialJoinResolver Partial join hql: " + hql); - String partSQL = - " left outer join " + getDbName() + "c1_citytable citydim on testcube.cityid " - + "= citydim.id and (( citydim . name ) = 'FOOBAR' ) " + "and (citydim.dt = 'latest')"; - Assert.assertTrue(hql.contains(partSQL)); - partSQL = - " right outer join " + getDbName() + "c1_testdim2tbl testdim2 on " - + "testcube.dim2 = testdim2.id right outer join " + getDbName() - + "c1_testdim3tbl testdim3 on testdim2.testdim3id = testdim3.id and " - + "(testdim2.dt = 'latest') right outer join " + getDbName() - + "c1_testdim4tbl testdim4 on testdim3.testdim4id = testdim4.id and " - + "(( testdim4 . name ) = 'TESTDIM4NAME' ) and (testdim3.dt = 'latest')"; - - Assert.assertTrue(hql.contains(partSQL)); + "SELECT citydim.name, testDim4.name, msr2 FROM testCube " + + " left outer join citydim ON testcube.cityid = citydim .id and citydim.name = 'FOOBAR'" + + " right outer join testdim2 on testcube.dim2 = testdim2.id " + + " right outer join testdim3 on testdim2.testdim3id = testdim3.id " + + " right outer join testDim4 on testdim3.testdim4id = testdim4.id and testDim4.name='TESTDIM4NAME'" + + " WHERE " + TWO_DAYS_RANGE; + String hqlQuery = rewrite(query, hconf); + String expected = getExpectedQuery("testcube", "select citydim.name, testDim4.name, sum(testcube.msr2) FROM ", + " left outer JOIN " + getDbName() + "c1_citytable citydim on testcube.cityid = citydim.id +" + + " and (( citydim . name ) = 'FOOBAR' ) and (citydim.dt = 'latest')" + + " right outer join " + getDbName() + + "c1_testdim2tbl testdim2 on testcube.dim2 = testdim2.id and (testdim2.dt = 'latest')" + + " right outer join " + getDbName() + "c1_testdim3tbl testdim3 on testdim2.testdim3id = testdim3.id and " + + "(testdim3.dt = 'latest') " + + " right outer join " + getDbName() + "c1_testdim4tbl testdim4 on testdim3.testdim4id = testdim4.id and " + + "(( testdim4 . name ) = 'TESTDIM4NAME' ) and (testdim4.dt = 'latest')", + null, "group by citydim.name, testdim4.name", null, + getWhereForDailyAndHourly2days("testcube", "c1_summary3")); + TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test @@ -258,122 +157,162 @@ public class TestJoinResolver extends TestQueryRewrite { @Test public void testJoinWithoutCondition() throws Exception { - String query = "SELECT citydim.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; - CubeQueryRewriter driver = new CubeQueryRewriter(hconf, hconf); - CubeQueryContext ctx = driver.rewrite(query); - String hql = ctx.toHQL(); - String joinClause = getAutoResolvedFromString(ctx); - System.out.println("@Resolved join clause " + joinClause); - Assert.assertEquals(getDbName() + "c1_testfact2_raw testcube join " + getDbName() + "c1_citytable citydim on " - + "testcube.cityid = citydim.id and (citydim.dt = 'latest')", joinClause.trim()); + assertLensExceptionInRewrite("SELECT citydim.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE, hconf, + LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); + assertLensExceptionInRewrite("select cubeState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE, + hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); + assertLensExceptionInRewrite("select citydim.name, statedim.name from citydim limit 10", + hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); + assertLensExceptionInRewrite("select countrydim.name, citystate.name from citydim limit 10", + hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); } + @Test public void testJoinTypeConf() throws Exception { HiveConf tConf = new HiveConf(hconf); tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "LEFTOUTER"); - System.out.println("@@Set join type to " + hconf.get(CubeQueryConfUtil.JOIN_TYPE_KEY)); - CubeQueryRewriter driver = new CubeQueryRewriter(tConf, hconf); - String query = "select citydim.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; - CubeQueryContext ctx = driver.rewrite(query); - String hql = ctx.toHQL(); - System.out.println("testJoinTypeConf@@Resolved join clause1 - " + getAutoResolvedFromString(ctx)); - Assert.assertEquals(getDbName() + "c1_testfact2_raw testcube left outer join " + getDbName() - + "c1_citytable citydim on testcube.cityid = citydim.id and (citydim.dt = 'latest')", - getAutoResolvedFromString(ctx).trim()); + String query = "select cubecity.name, msr2 FROM testCube WHERE " + TWO_DAYS_RANGE; + String hqlQuery = rewrite(query, tConf); + // Check that aliases are preserved in the join clause + String expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", + " left outer join " + getDbName() + + "c1_citytable cubecity ON testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", + null, " group by cubecity.name", null, getWhereForHourly2days("testcube", "c1_testfact2")); + TestCubeRewriter.compareQueries(hqlQuery, expected); tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "FULLOUTER"); - System.out.println("@@Set join type to " + hconf.get(CubeQueryConfUtil.JOIN_TYPE_KEY)); - driver = new CubeQueryRewriter(tConf, hconf); - ctx = driver.rewrite(query); - hql = ctx.toHQL(); - System.out.println("testJoinTypeConf@@Resolved join clause2 - " + getAutoResolvedFromString(ctx)); - Assert.assertEquals(getDbName() + "c1_testfact2_raw testcube full outer join " + getDbName() - + "c1_citytable citydim on testcube.cityid = citydim.id and (citydim.dt = 'latest')", - getAutoResolvedFromString(ctx).trim()); + hqlQuery = rewrite(query, tConf); + // Check that aliases are preserved in the join clause + expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", + " full outer join " + getDbName() + + "c1_citytable cubecity ON testcube.cityid = cubecity.id and (cubecity.dt = 'latest')", + null, " group by cubecity.name", null, getWhereForHourly2days("testcube", "c1_testfact2")); + TestCubeRewriter.compareQueries(hqlQuery, expected); + + tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "RIGHTOUTER"); + hqlQuery = rewrite(query, tConf); + // Check that aliases are preserved in the join clause + expected = getExpectedQuery("testcube", "select cubecity.name, sum(testcube.msr2) FROM ", + " right outer join " + getDbName() + + "c1_citytable cubecity ON testcube.cityid = cubecity.id", + null, " and (cubecity.dt = 'latest') group by cubecity.name", null, + getWhereForHourly2days("testcube", "c1_testfact2")); + TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test - public void testPreserveTableAlias() throws Exception { + public void testPreserveTableAliasWithFullJoin() throws Exception { HiveConf tConf = new HiveConf(hconf); tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "LEFTOUTER"); - String query = "select c.name, t.msr2 FROM testCube t join citydim c WHERE " + TWO_DAYS_RANGE; - CubeQueryRewriter driver = new CubeQueryRewriter(tConf, hconf); - CubeQueryContext ctx = driver.rewrite(query); - String hql = ctx.toHQL(); - System.out.println("testPreserveTableAlias@@HQL:" + hql); - System.out.println("testPreserveTableAlias@@Resolved join clause - " + getAutoResolvedFromString(ctx)); + String query = "select c.name, t.msr2 FROM testCube t join citydim c on t.cityid = c.id WHERE " + TWO_DAYS_RANGE; + String hqlQuery = rewrite(query, tConf); + // Check that aliases are preserved in the join clause + // Conf will be ignored in this case since user has specified the join condition + String expected = getExpectedQuery("t", "select c.name, sum(t.msr2) FROM ", + " inner join " + getDbName() + "c1_citytable c ON t.cityid = c.id and c.dt = 'latest'", + null, " group by c.name", null, getWhereForHourly2days("t", "c1_testfact2")); + TestCubeRewriter.compareQueries(hqlQuery, expected); + } + + @Test + public void testPreserveTableAliasWithAutoJoin() throws Exception { + HiveConf tConf = new HiveConf(hconf); + tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "LEFTOUTER"); + String query = "select cubecity.name, t.msr2 FROM testCube t WHERE " + TWO_DAYS_RANGE; + String hqlQuery = rewrite(query, tConf); // Check that aliases are preserved in the join clause - // Conf will be ignored in this case since user has specified partial join - Assert.assertEquals(getDbName() + "c1_testfact2_raw t inner join " + getDbName() - + "c1_citytable c on t.cityid = c.id and (c.dt = 'latest')", getAutoResolvedFromString(ctx).trim()); - String whereClause = hql.substring(hql.indexOf("WHERE")); - // Check that the partition condition is not added again in where clause - Assert.assertFalse(whereClause.contains("c.dt = 'latest'")); + String expected = getExpectedQuery("t", "select cubecity.name, sum(t.msr2) FROM ", + " left outer join " + getDbName() + + "c1_citytable cubecity ON t.cityid = cubecity.id and (cubecity.dt = 'latest')", + null, " group by cubecity.name", null, getWhereForHourly2days("t", "c1_testfact2")); + TestCubeRewriter.compareQueries(hqlQuery, expected); } @Test - public void testDimOnlyQuery() throws Exception { + public void testDimOnlyQueryWithAutoJoin() throws Exception { HiveConf tConf = new HiveConf(hconf); tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "INNER"); - String query = "select citydim.name, statedim.name from citydim limit 10"; - HiveConf dimOnlyConf = new HiveConf(tConf); - CubeQueryRewriter rewriter = new CubeQueryRewriter(dimOnlyConf, hconf); - CubeQueryContext ctx = rewriter.rewrite(query); - String hql = ctx.toHQL(); - System.out.println("testDimOnlyQuery@@@HQL:" + hql); - System.out.println("testDimOnlyQuery@@@Resolved join clause: " + getAutoResolvedFromString(ctx)); - Assert.assertTrue(hql.matches(".*?WHERE\\W+citydim.dt = 'latest'\\W+LIMIT 10.*?")); - Assert.assertEquals(getDbName() + "c1_citytable citydim inner join " + getDbName() - + "c1_statetable statedim on citydim.stateid = statedim.id and (statedim.dt = 'latest')", - getAutoResolvedFromString(ctx).trim()); - - String queryWithJoin = "select citydim.name, statedim.name from citydim join statedim"; - ctx = rewriter.rewrite(queryWithJoin); - hql = ctx.toHQL(); - System.out.println("testDimOnlyQuery@@@HQL2:" + hql); - HQLParser.parseHQL(hql, tConf); - Assert.assertEquals(getDbName() + "c1_citytable citydim inner join " + getDbName() - + "c1_statetable statedim on citydim.stateid = statedim.id and (statedim.dt = 'latest')", - getAutoResolvedFromString(ctx).trim()); + String query = "select citydim.name, citystate.name from citydim limit 10"; + String hqlQuery = rewrite(query, tConf); + String expected = + getExpectedQuery("citydim", "select citydim.name, citystate.name from ", " inner join " + getDbName() + + "c1_statetable citystate on citydim.stateid = citystate.id and (citystate.dt = 'latest')", + null, " limit 10", "c1_citytable", true); + compareQueries(hqlQuery, expected); } @Test - public void testStorageFilterPushdown() throws Exception { - String q = "SELECT citydim.name, statedim.name FROM citydim"; + public void testDimOnlyQueryWithFullJoin() throws Exception { + HiveConf tConf = new HiveConf(hconf); + tConf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "INNER"); + String queryWithJoin = "select citydim.name, statedim.name from citydim join statedim on citydim.stateid = " + + "statedim.id"; + + String hqlQuery = rewrite(queryWithJoin, tConf); + String expected = + getExpectedQuery("citydim", "select citydim.name, statedim.name from ", " inner join " + getDbName() + + "c1_statetable statedim on citydim.stateid = statedim.id and citydim.dt='latest' and statedim.dt='latest'", + null, null, "c1_citytable", false); + compareQueries(hqlQuery, expected); + } + + @Test + public void testStorageFilterPushdownWithFullJoin() throws Exception { + String q1 = "SELECT citydim.name, statedim.name FROM citydim left outer join statedim on citydim.stateid = " + + "statedim.id"; + String hqlQuery = rewrite(q1, hconf); + String expected = + getExpectedQuery("citydim", "select citydim.name, statedim.name from ", " left outer join " + getDbName() + + "c1_statetable statedim on citydim.stateid = statedim.id and citydim.dt='latest' and statedim.dt='latest'", + null, null, "c1_citytable", false); + compareQueries(hqlQuery, expected); + + String q2 = "SELECT citydim.name, statedim.name FROM citydim right outer join statedim on citydim.stateid = " + + "statedim.id"; + hqlQuery = rewrite(q2, hconf); + expected = + getExpectedQuery("citydim", "select citydim.name, statedim.name from ", " right outer join " + getDbName() + + "c1_statetable statedim on citydim.stateid = statedim.id and citydim.dt='latest' and statedim.dt='latest'", + null, null, "c1_citytable", false); + compareQueries(hqlQuery, expected); + + String q3 = "SELECT citydim.name, statedim.name FROM citydim full outer join statedim on citydim.stateid = " + + "statedim.id"; + hqlQuery = rewrite(q3, hconf); + expected = + getExpectedQuery("citydim", "select citydim.name, statedim.name from ", " full outer join " + getDbName() + + "c1_statetable statedim on citydim.stateid = statedim.id and citydim.dt='latest' and statedim.dt='latest'", + null, null, "c1_citytable", false); + } + + @Test + public void testStorageFilterPushdownWithAutoJoin() throws Exception { + String q = "SELECT citydim.name, citystate.name FROM citydim limit 10"; HiveConf conf = new HiveConf(hconf); conf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "LEFTOUTER"); - CubeQueryRewriter rewriter = new CubeQueryRewriter(conf, hconf); - CubeQueryContext context = rewriter.rewrite(q); - String hql = context.toHQL(); - System.out.println("##1 hql " + hql); - System.out.println("##1 " + getAutoResolvedFromString(context)); - Assert.assertEquals(getDbName() + "c1_citytable citydim left outer join " + getDbName() - + "c1_statetable statedim on citydim.stateid = statedim.id" + " and (statedim.dt = 'latest')", - getAutoResolvedFromString(context).trim()); - Assert.assertTrue(hql.matches(".*?WHERE\\W+citydim.dt = 'latest'\\W+.*?")); + String hqlQuery = rewrite(q, conf); + String expected = + getExpectedQuery("citydim", "select citydim.name, citystate.name from ", " left outer join " + getDbName() + + "c1_statetable citystate on citydim.stateid = citystate.id and (citystate.dt = 'latest')", + null, " limit 10", "c1_citytable", true); + compareQueries(hqlQuery, expected); conf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "RIGHTOUTER"); - rewriter = new CubeQueryRewriter(conf, hconf); - context = rewriter.rewrite(q); - hql = context.toHQL(); - System.out.println("##2 hql " + hql); - System.out.println("##2 " + getAutoResolvedFromString(context)); - Assert.assertEquals(getDbName() + "c1_citytable citydim right outer join " + getDbName() - + "c1_statetable statedim on citydim.stateid = statedim.id " + "and (citydim.dt = 'latest')", - getAutoResolvedFromString(context).trim()); - Assert.assertTrue(hql.matches(".*?WHERE\\W+statedim.dt = 'latest'\\W+.*?")); + hqlQuery = rewrite(q, conf); + expected = + getExpectedQuery("citydim", "select citydim.name, citystate.name from ", " right outer join " + getDbName() + + "c1_statetable citystate on citydim.stateid = citystate.id and (citydim.dt = 'latest')", + " citystate.dt='latest' ", "limit 10", "c1_citytable", false); + compareQueries(hqlQuery, expected); conf.set(CubeQueryConfUtil.JOIN_TYPE_KEY, "FULLOUTER"); - rewriter = new CubeQueryRewriter(conf, hconf); - context = rewriter.rewrite(q); - hql = context.toHQL(); - System.out.println("##3 hql " + hql); - System.out.println("##3 " + getAutoResolvedFromString(context)); - Assert.assertEquals(getDbName() + "c1_citytable citydim full outer join " + getDbName() - + "c1_statetable statedim on citydim.stateid = statedim.id " - + "and (citydim.dt = 'latest') and (statedim.dt = 'latest')", getAutoResolvedFromString(context).trim()); - Assert.assertTrue(!hql.contains("WHERE")); + hqlQuery = rewrite(q, conf); + expected = + getExpectedQuery("citydim", "select citydim.name, citystate.name from ", " full outer join " + getDbName() + + "c1_statetable citystate on citydim.stateid = citystate.id and (citydim.dt = 'latest')" + + " and citystate.dt='latest'", null, "limit 10", "c1_citytable", false); + compareQueries(hqlQuery, expected); } @Test @@ -434,14 +373,14 @@ public class TestJoinResolver extends TestQueryRewrite { null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base")); TestCubeRewriter.compareQueries(hqlQuery, expected); - // Single join chain and an unrelated dimension - query = "select cubeState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; + // Two unrelated join chains + query = "select cubeState.name, cubecity.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); expected = getExpectedQuery("basecube", - "select cubestate.name, citydim.name, sum(basecube.msr2) FROM ", + "select cubestate.name, cubecity.name, sum(basecube.msr2) FROM ", " join " + getDbName() + "c1_statetable cubestate on basecube.stateid = cubestate.id and cubestate.dt = 'latest'" - + " join " + getDbName() + "c1_citytable citydim on basecube.cityid = citydim.id and citydim.dt = 'latest'", - null, "group by cubestate.name,citydim.name", null, + + " join " + getDbName() + "c1_citytable cubecity on basecube.cityid = cubecity.id and cubecity.dt = 'latest'", + null, "group by cubestate.name,cubecity.name", null, getWhereForDailyAndHourly2days("basecube", "c1_testfact1_base") ); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -580,149 +519,110 @@ public class TestJoinResolver extends TestQueryRewrite { public void testConflictingJoins() throws ParseException, LensException, HiveException { // Single joinchain with two paths, intermediate dimension accessed separately by name. String query = "select cityState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; - try { - rewrite(query, hconf); - Assert.fail("Should have failed. " - + "The table citydim is getting accessed as both chain and without chain "); - } catch (LensException e) { - Assert.assertEquals(e.getMessage().toLowerCase(), - "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase()); - } + assertLensExceptionInRewrite(query, hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); // Multi joinchains + a dimension part of one of the chains. query = "select cityState.name, cubeState.name, citydim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; - try { - rewrite(query, hconf); - Assert.fail("Should have failed. " - + "The table citydim is getting accessed as both chain and without chain "); - } catch (LensException e) { - Assert.assertEquals(e.getMessage().toLowerCase(), - "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase()); - } + assertLensExceptionInRewrite(query, hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); // this test case should pass when default qualifiers for dimensions' chains are added // Two joinchains with same destination, and the destination table accessed separately query = "select cityState.name, cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; - try { - rewrite(query, hconf); - Assert.fail("Should have failed. " - + "It's not possible to resolve which statedim is being asked for when cityState and cubeState both end at" - + " statedim table."); - } catch (LensException e) { - Assert.assertEquals( - e.getMessage().indexOf("Table statedim has 2 different paths through joinchains"), 0); - } + assertLensExceptionInRewrite(query, hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); // this test case should pass when default qualifiers for dimensions' chains are added // Two Single joinchain, And dest table accessed separately. query = "select cubeState.name, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; - try { - rewrite(query, hconf); - Assert.fail("Should have failed. " - + "The table statedim is getting accessed as both cubeState and statedim "); - } catch (LensException e) { - Assert.assertEquals(e.getMessage().toLowerCase(), - "Table statedim is getting accessed via two different names: [cubestate, statedim]".toLowerCase()); - } + assertLensExceptionInRewrite(query, hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); + // this should pass when default qualifiers are added query = "select cityStateCapital, statedim.name, sum(msr2) from basecube where " + TWO_DAYS_RANGE; - try { - rewrite(query, hconf); - Assert.fail("Should have failed. " - + "The table statedim is getting accessed as both cubeState and statedim "); - } catch (LensException e) { - Assert.assertEquals(e.getMessage().toLowerCase(), - "Table statedim is getting accessed via two different names: [citystate, statedim]".toLowerCase()); - } + assertLensExceptionInRewrite(query, hconf, LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE); // table accessed through denorm column and chain column Configuration conf = new Configuration(hconf); conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3, C4"); String failingQuery = "select testDim2.cityname, testDim2.cityStateCapital FROM testDim2 where " + TWO_DAYS_RANGE; - try { - rewrite(failingQuery, conf); - Assert.fail("Should have failed. " - + "The table citydim is getting accessed as both chain and without chain "); - } catch (LensException e) { - Assert.assertEquals(e.getMessage().toLowerCase(), - "Table citydim is getting accessed via joinchain: citystate and no chain at all".toLowerCase()); - } + assertLensExceptionInRewrite(failingQuery, conf, LensCubeErrorCode.NO_REF_COL_AVAILABLE); } @Test public void testMultiPaths() throws ParseException, LensException, HiveException { String query, hqlQuery, expected; - query = "select testdim3.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select dim3chain.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim3.name, sum(testcube.msr2) FROM ", - " join " + getDbName() + "c1_testdim3tbl testdim3 ON testcube.testdim3id=testdim3.id and testdim3.dt='latest'", - null, "group by testdim3.name", + expected = getExpectedQuery("testcube", "select dim3chain.name, sum(testcube.msr2) FROM ", + " join " + getDbName() + "c1_testdim3tbl dim3chain ON testcube.testdim3id=dim3chain.id and dim3chain.dt='latest'", + null, "group by dim3chain.name", null, getWhereForDailyAndHourly2days("testcube", "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); // hit a fact where there is no direct path - query = "select testdim3.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select dim3chain.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim3.name, avg(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim3chain.name, avg(testcube.msr2) FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" - + " join " + getDbName() + "c1_testdim3tbl testdim3 " - + "ON testdim2.testdim3id = testdim3.id and testdim3.dt = 'latest'", - null, "group by testdim3.name", + + " join " + getDbName() + "c1_testdim3tbl dim3chain " + + "ON testdim2.testdim3id = dim3chain.id and dim3chain.dt = 'latest'", + null, "group by dim3chain.name", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); // resolve denorm variable through multi hop chain paths query = "select testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim3.id, avg(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim3chain.id, avg(testcube.msr2) FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" - + " join " + getDbName() + "c1_testdim3tbl testdim3 " - + "ON testdim2.testdim3id = testdim3.id and testdim3.dt = 'latest'", - null, "group by testdim3.id", + + " join " + getDbName() + "c1_testdim3tbl dim3chain " + + "ON testdim2.testdim3id = dim3chain.id and dim3chain.dt = 'latest'", + null, "group by dim3chain.id", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); // tests from multiple different chains - query = "select testdim4.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select dim4chain.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim4.name, testdim3.id, avg(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim4chain.name, dim3chain.id, avg(testcube.msr2) FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" - + " join " + getDbName() + "c1_testdim3tbl testdim3 ON testdim2.testdim3id=testdim3.id and testdim3.dt='latest'" - + " join " + getDbName() + "c1_testdim4tbl testdim4 ON testdim3.testDim4id = testdim4.id and" - + " testdim4.dt = 'latest'", null, "group by testdim4.name, testdim3.id", null, + + " join " + getDbName() + + "c1_testdim3tbl dim3chain ON testdim2.testdim3id=dim3chain.id and dim3chain.dt='latest'" + + " join " + getDbName() + "c1_testdim4tbl dim4chain ON dim3chain.testDim4id = dim4chain.id and" + + " dim4chain.dt = 'latest'", null, "group by dim4chain.name, dim3chain.id", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); - query = "select citydim.name, testdim4.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select cubecity.name, dim4chain.name, testdim3id, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select citydim.name, testdim4.name, testdim3.id, avg(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select cubecity.name, dim4chain.name, dim3chain.id, avg(testcube.msr2) " + + "FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" - + " join " + getDbName() + "c1_testdim3tbl testdim3 ON testdim2.testdim3id=testdim3.id and testdim3.dt='latest'" - + " join " + getDbName() + "c1_testdim4tbl testdim4 ON testdim3.testDim4id = testdim4.id and" - + " testdim4.dt = 'latest'" - + " join " + getDbName() + "c1_citytable citydim ON testcube.cityid = citydim.id and citydim.dt = 'latest'" - , null, "group by citydim.name, testdim4.name, testdim3.id", null, + + " join " + getDbName() + + "c1_testdim3tbl dim3chain ON testdim2.testdim3id=dim3chain.id and dim3chain.dt='latest'" + + " join " + getDbName() + "c1_testdim4tbl dim4chain ON dim3chain.testDim4id = dim4chain.id and" + + " dim4chain.dt = 'latest'" + + " join " + getDbName() + "c1_citytable cubecity ON testcube.cityid = cubecity.id and cubecity.dt = 'latest'" + , null, "group by cubecity.name, dim4chain.name, dim3chain.id", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); // test multi hops - query = "select testdim4.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select dim4chain.name, avg(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim4.name, avg(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim4chain.name, avg(testcube.msr2) FROM ", " join " + getDbName() + "c1_testdim2tbl testdim2 ON testcube.dim2 = testdim2.id and testdim2.dt = 'latest'" + " join " + getDbName() + "c1_testdim3tbl testdim3 ON testdim2.testdim3id=testdim3.id and testdim3.dt='latest'" - + " join " + getDbName() + "c1_testdim4tbl testdim4 ON testdim3.testDim4id = testdim4.id and" - + " testdim4.dt = 'latest'", null, "group by testdim4.name", null, + + " join " + getDbName() + "c1_testdim4tbl dim4chain ON testdim3.testDim4id = dim4chain.id and" + + " dim4chain.dt = 'latest'", null, "group by dim4chain.name", null, getWhereForHourly2days("testcube", "c1_testfact2_raw")); TestCubeRewriter.compareQueries(hqlQuery, expected); - query = "select testdim4.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; + query = "select dim4chain.name, sum(msr2) from testcube where " + TWO_DAYS_RANGE; hqlQuery = rewrite(query, hconf); - expected = getExpectedQuery("testcube", "select testdim4.name, sum(testcube.msr2) FROM ", + expected = getExpectedQuery("testcube", "select dim4chain.name, sum(testcube.msr2) FROM ", " join " + getDbName() + "c1_testdim3tbl testdim3 ON testcube.testdim3id = testdim3.id and testdim3.dt = 'latest'" - + " join " + getDbName() + "c1_testdim4tbl testdim4 ON testdim3.testDim4id = testdim4.id and" - + " testdim4.dt = 'latest'", null, "group by testdim4.name", null, + + " join " + getDbName() + "c1_testdim4tbl dim4chain ON testdim3.testDim4id = dim4chain.id and" + + " dim4chain.dt = 'latest'", null, "group by dim4chain.name", null, getWhereForDailyAndHourly2days("testcube", "c1_summary1")); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -748,13 +648,17 @@ public class TestJoinResolver extends TestQueryRewrite { @Test public void testUnreachableDim() throws ParseException, LensException, HiveException { - LensException e1 = getLensExceptionInRewrite("select urdimid from testdim2", hconf); - assertNotNull(e1); - assertEquals(e1.getErrorCode(), LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo().getErrorCode()); - - LensException e2 = getLensExceptionInRewrite("select urdimid from testcube where " + TWO_DAYS_RANGE, hconf); - assertNotNull(e2); - assertEquals(e2.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); + assertLensExceptionInRewrite("select urdimid from testdim2", hconf, LensCubeErrorCode.NO_DIM_HAS_COLUMN); + assertLensExceptionInRewrite("select urdimid from testcube where " + TWO_DAYS_RANGE, hconf, + LensCubeErrorCode.NO_FACT_HAS_COLUMN); + assertLensExceptionInRewrite("select unreachableName from testdim2", hconf, + LensCubeErrorCode.NO_DIM_HAS_COLUMN); + assertLensExceptionInRewrite("select unreachableName from testcube where " + TWO_DAYS_RANGE, hconf, + LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE); + assertLensExceptionInRewrite("select unreachableDim_chain.name from testdim2", hconf, + LensCubeErrorCode.NO_JOIN_PATH); + assertLensExceptionInRewrite("select unreachableDim_chain.name from testcube where " + TWO_DAYS_RANGE, hconf, + LensCubeErrorCode.NO_FACT_HAS_COLUMN); } @Test
http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java index d69635d..0aa31f4 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java @@ -19,11 +19,15 @@ package org.apache.lens.cube.parse; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + import java.io.IOException; import org.apache.lens.api.error.ErrorCollection; import org.apache.lens.api.error.ErrorCollectionFactory; import org.apache.lens.api.error.LensError; +import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.conf.Configuration; @@ -114,6 +118,12 @@ public abstract class TestQueryRewrite { } } + protected void assertLensExceptionInRewrite(String query, Configuration conf, LensCubeErrorCode expectedError) + throws LensException, ParseException { + LensException e = getLensExceptionInRewrite(query, conf); + assertNotNull(e); + assertEquals(e.getErrorCode(), expectedError.getLensErrorInfo().getErrorCode()); + } protected String getLensExceptionErrorMessageInRewrite(String query, Configuration conf) throws LensException, ParseException, ClassNotFoundException { try { http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java index 5a072e4..2d7babb 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestRewriterPlan.java @@ -67,8 +67,9 @@ public class TestRewriterPlan extends TestQueryRewrite { public void testPlanExtractionForComplexQuery() throws Exception { // complex query Configuration conf = getConfWithStorages("C1,C2"); - CubeQueryContext ctx = rewriteCtx("cube select citydim.name, SUM(msr2) from testCube where citydim.name != \"XYZ\"" - + " and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by citydim.name limit 50", conf); + CubeQueryContext ctx = rewriteCtx("cube select cubecity.name, SUM(msr2) from testCube where " + + " cubecity.name != \"XYZ\" and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by cubecity.name limit 50", + conf); ctx.toHQL(); RewriterPlan plan = new RewriterPlan(Collections.singleton(ctx)); Assert.assertNotNull(plan); @@ -90,8 +91,9 @@ public class TestRewriterPlan extends TestQueryRewrite { Configuration conf = getConfWithStorages("C1,C2"); CubeQueryContext ctx1 = rewriteCtx("cube select SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf); ctx1.toHQL(); - CubeQueryContext ctx2 = rewriteCtx("cube select citydim.name, SUM(msr2) from testCube where citydim.name != \"XYZ\"" - + " and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by citydim.name limit 50", conf); + CubeQueryContext ctx2 = rewriteCtx("cube select cubecity.name, SUM(msr2) from testCube where " + + " cubecity.name != \"XYZ\" and " + TWO_DAYS_RANGE + " having sum(msr2) > 1000 order by cubecity.name limit 50", + conf); ctx2.toHQL(); RewriterPlan plan = new RewriterPlan(Arrays.asList(ctx1, ctx2)); Assert.assertNotNull(plan); http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java index b7372f1..a0ee56d 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java @@ -154,14 +154,14 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { "SELECT test_time_dim, msr2 FROM testCube where " + TWO_DAYS_RANGE_TTD; String hqlQuery = rewrite(query, tconf); Map<String, String> whereClauses = new HashMap<String, String>(); - whereClauses.put(getDbName() + "c4_testfact2", TestBetweenTimeRangeWriter.getBetweenClause("hourdim", + whereClauses.put(getDbName() + "c4_testfact2", TestBetweenTimeRangeWriter.getBetweenClause("timehourchain1", "full_hour", getUptoHour(TWODAYS_BACK), getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); System.out.println("HQL:" + hqlQuery); String expected = - getExpectedQuery(cubeName, "select hourdim.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, - " GROUP BY hourdim.full_hour", null, whereClauses); + getExpectedQuery(cubeName, "select timehourchain1.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() + + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, + " GROUP BY timehourchain1.full_hour", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); query = @@ -170,7 +170,8 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, null, null, whereClauses); + + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, null, null, + whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); query = @@ -179,7 +180,8 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", " testcube.cityid > 2 ", + + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", + " testcube.cityid > 2 ", " and testcube.cityid != 5", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -192,16 +194,17 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { whereClauses = new HashMap<>(); whereClauses.put( getDbName() + "c4_testfact2", - TestBetweenTimeRangeWriter.getBetweenClause("hourdim", "full_hour", getUptoHour(TWODAYS_BACK), + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain1", "full_hour", getUptoHour(TWODAYS_BACK), getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT) + " OR " - + TestBetweenTimeRangeWriter.getBetweenClause("hourdim", "full_hour", getUptoHour(BEFORE_6_DAYS), + + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain1", "full_hour", getUptoHour(BEFORE_6_DAYS), getUptoHour(getOneLess(BEFORE_4_DAYS, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, null, null, whereClauses); + + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, null, null, + whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -211,9 +214,9 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { + " OR " + TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS, tconf); expected = - getExpectedQuery(cubeName, "select to_date(hourdim.full_hour), sum(testcube.msr2) FROM ", " join " - + getDbName() + "c4_hourDimTbl hourdim on testcube.test_time_dim_hour_id = hourdim.id", null, - " group by to_date(hourdim.full_hour)", null, whereClauses); + getExpectedQuery(cubeName, "select to_date(timehourchain1.full_hour), sum(testcube.msr2) FROM ", " join " + + getDbName() + "c4_hourDimTbl timehourchain1 on testcube.test_time_dim_hour_id = timehourchain1.id", null, + " group by to_date(timehourchain1.full_hour)", null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); } @@ -233,13 +236,13 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { String hqlQuery = rewrite(query, tconf); Map<String, String> whereClauses = new HashMap<String, String>(); whereClauses.put(getDbName() + "c4_testfact2", TestBetweenTimeRangeWriter.getBetweenClause( - "timehourchain", "full_hour", getUptoHour(TWODAYS_BACK), + "timehourchain2", "full_hour", getUptoHour(TWODAYS_BACK), getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); System.out.println("HQL:" + hqlQuery); String expected = - getExpectedQuery(cubeName, "select timehourchain.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, - " GROUP BY timehourchain.full_hour", null, whereClauses); + getExpectedQuery(cubeName, "select timehourchain2.full_hour, sum(testcube.msr2) FROM ", " join " + getDbName() + + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, + " GROUP BY timehourchain2.full_hour", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); query = @@ -248,7 +251,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, null, null, + + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, null, null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -258,7 +261,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", + + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", " testcube.cityid > 2 ", " and testcube.cityid != 5", null, whereClauses); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -268,19 +271,19 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { "select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE_TTD2 + " OR " + TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS, tconf); - whereClauses = new HashMap<String, String>(); + whereClauses = new HashMap<>(); whereClauses.put( getDbName() + "c4_testfact2", - TestBetweenTimeRangeWriter.getBetweenClause("timehourchain", "full_hour", getUptoHour(TWODAYS_BACK), + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain2", "full_hour", getUptoHour(TWODAYS_BACK), getUptoHour(getOneLess(NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT) + " OR " - + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain", "full_hour", getUptoHour(BEFORE_6_DAYS), + + TestBetweenTimeRangeWriter.getBetweenClause("timehourchain2", "full_hour", getUptoHour(BEFORE_6_DAYS), getUptoHour(getOneLess(BEFORE_4_DAYS, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT)); expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", " join " + getDbName() - + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, null, null, + + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, null, null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); @@ -291,9 +294,9 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite { + " OR " +TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS, tconf); expected = - getExpectedQuery(cubeName, "select to_date(timehourchain.full_hour), sum(testcube.msr2) FROM ", " join " - + getDbName() + "c4_hourDimTbl timehourchain on testcube.test_time_dim_hour_id2 = timehourchain.id", null, - " group by to_date(timehourchain.full_hour)", null, whereClauses); + getExpectedQuery(cubeName, "select to_date(timehourchain2.full_hour), sum(testcube.msr2) FROM ", " join " + + getDbName() + "c4_hourDimTbl timehourchain2 on testcube.test_time_dim_hour_id2 = timehourchain2.id", null, + " group by to_date(timehourchain2.full_hour)", null, whereClauses); System.out.println("HQL:" + hqlQuery); TestCubeRewriter.compareQueries(hqlQuery, expected); } http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/cube-queries.sql ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/cube-queries.sql b/lens-examples/src/main/resources/cube-queries.sql index 13c5204..9f4a353 100644 --- a/lens-examples/src/main/resources/cube-queries.sql +++ b/lens-examples/src/main/resources/cube-queries.sql @@ -26,33 +26,33 @@ cube select dim1, measure2 from sample_cube where time_range_in(dt, '2014-06-24- cube select dim3, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') cube select dim3, measure3 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') cube select dim3, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') order by sample_dim.name -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') order by sample_dim.name -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') order by sample_dim.name -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" order by sample_dim.name -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" order by sample_dim.name desc -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" order by sample_dim.name -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" order by sample_dim.name desc -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" order by sample_dim.name -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" order by sample_dim.name desc -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') order by sample_dim.name limit 2 -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') order by sample_dim.name limit 2 -cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') order by sample_dim.name limit 2 -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" limit 2 -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" limit 2 -cube select sample_dim.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" order by sample_dim.name limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim.name != "first" order by sample_dim.name desc limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" order by sample_dim.name limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim.name != "first" order by sample_dim.name desc limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" order by sample_dim.name limit 2 -cube select sample_dim.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim.name != "first" order by sample_dim.name desc limit 2 +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') order by sample_dim_chain.name +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') order by sample_dim_chain.name +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') order by sample_dim_chain.name +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" order by sample_dim_chain.name +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" limit 2 +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" limit 2 +cube select sample_dim_chain.name, measure3 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" order by sample_dim_chain.name limit 2 +cube select sample_dim_chain.name, measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') and sample_dim_chain.name != "first" order by sample_dim_chain.name desc limit 2 cube select dim1, dim2, measure1, measure2 from sample_cube where time_range_in(dt, '2014-06-25-20', '2014-06-26-02') cube select dim1, dim2, measure1, measure2 from sample_cube where time_range_in(dt, '2014-06-25-20', '2014-06-26-02') order by dim2 cube select dim1, dim2, measure1, measure2 from sample_cube where time_range_in(dt, '2014-06-25-20', '2014-06-26-02') order by dim2 desc @@ -67,23 +67,23 @@ cube select dim1, sum(measure2) from sample_cube where time_range_in(dt, '2014-0 cube select dim3, max(measure3) from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') cube select dim3, max(measure3) from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') cube select dim3, max(measure3) from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') -cube select sample_dim.name, measure4 from sample_cube join sample_dim where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') -cube select sample_dim.name, measure4 from sample_cube left outer join sample_dim where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') -cube select sample_dim.name, measure4 from sample_cube right outer join sample_dim where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') -cube select sample_dim.name, measure4 from sample_cube full outer join sample_dim where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') -select * from (cube select sample_dim.name, measure4 from sample_cube join sample_dim where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') ) a +cube select sample_dim.name, measure4 from sample_cube join sample_dim on sample_cube.dim3=sample_dim.id where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +cube select sample_dim.name, measure4 from sample_cube left outer join sample_dim on sample_cube.dim3=sample_dim.id where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') +cube select sample_dim.name, measure4 from sample_cube right outer join sample_dim on sample_cube.dim3=sample_dim.id where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') +cube select sample_dim.name, measure4 from sample_cube full outer join sample_dim on sample_cube.dim3=sample_dim.id where time_range_in(dt, '2014-06-24-23', '2014-06-26-01') +select * from (cube select sample_dim.name, measure4 from sample_cube join sample_dim on sample_cube.dim3=sample_dim.id where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') ) a drop table temp1 -create table temp1 as cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +create table temp1 as cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') select * from temp1 -insert overwrite local directory '/tmp/example-cube-output' cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') -insert overwrite local directory '/tmp/example-cube-output2' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ('serialization.null.format'='-NA-','field.delim'=',' ) STORED AS TEXTFILE cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +insert overwrite local directory '/tmp/example-cube-output' cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +insert overwrite local directory '/tmp/example-cube-output2' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ('serialization.null.format'='-NA-','field.delim'=',' ) STORED AS TEXTFILE cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') drop table temp2 create table temp2(name string, msr4 float) -insert overwrite table temp2 cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +insert overwrite table temp2 cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') select * from temp2 drop table temp3 create table temp3(name string, msr4 float) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' WITH SERDEPROPERTIES ('serialization.null.format'='-NA-','field.delim'=',' ) STORED AS TEXTFILE -insert overwrite table temp3 cube select sample_dim.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') +insert overwrite table temp3 cube select sample_dim_chain.name, measure4 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00') select * from temp3 cube select product_id, store_sales from sales where time_range_in(order_time, '2015-04-11-00', '2015-04-13-00') cube select product_id, store_sales from sales where time_range_in(order_time, '2015-04-11-00', '2015-04-13-01') http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/customer.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/customer.xml b/lens-examples/src/main/resources/customer.xml index 920fd49..c6182a2 100644 --- a/lens-examples/src/main/resources/customer.xml +++ b/lens-examples/src/main/resources/customer.xml @@ -32,9 +32,7 @@ <dim_attribute name="city_id" type="INT" /> <dim_attribute name="customer_city_name" type="string" description="City name to which the customer belongs" display_string="Customer City"> - <ref_spec> - <chain_ref_column chain_name="customer_city" ref_col="name" /> - </ref_spec> + <chain_ref_column chain_name="customer_city" ref_col="name" /> </dim_attribute> <dim_attribute name="customer_credit_status" type="STRING" start_time='2015-03-01T00:00:00'/> </attributes> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/dimension-queries.sql ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dimension-queries.sql b/lens-examples/src/main/resources/dimension-queries.sql index 7aff639..a5f51d9 100644 --- a/lens-examples/src/main/resources/dimension-queries.sql +++ b/lens-examples/src/main/resources/dimension-queries.sql @@ -32,14 +32,14 @@ cube select id,name from sample_dim where name != 'first' order by name desc lim cube select count(id) from sample_dim cube select count(id) from sample_dim group by name cube select count(distinct id) from sample_dim -cube select sample_dim.name, sample_dim2.name from sample_dim -cube select sample_dim.name, sample_dim2.name from sample_dim join sample_dim2 -cube select sample_dim.name, sample_dim2.name from sample_dim left outer join sample_dim2 -cube select sample_dim.name, sample_dim2.name from sample_dim right outer join sample_dim2 -cube select sample_dim.name, sample_dim2.name from sample_dim full outer join sample_dim2 +cube select sample_dim.name, sample_dim2_chain.name from sample_dim +cube select sample_dim.name, sample_dim2.name from sample_dim join sample_dim2 on sample_dim.d2id=sample_dim2.id +cube select sample_dim.name, sample_dim2.name from sample_dim left outer join sample_dim2 on sample_dim.d2id=sample_dim2.id +cube select sample_dim.name, sample_dim2.name from sample_dim right outer join sample_dim2 on sample_dim.d2id=sample_dim2.id +cube select sample_dim.name, sample_dim2.name from sample_dim full outer join sample_dim2 on sample_dim.d2id=sample_dim2.id cube select count(id) from sample_dim where name != "first" cube select count(distinct id) from sample_dim where name != "first" -cube select sample_dim.name, sample_dim2.name from sample_dim where sample_dim.name != 'first' +cube select sample_dim.name, sample_dim2_chain.name from sample_dim where sample_dim.name != 'first' cube select id,name from sample_db_dim cube select id,name from sample_db_dim where name != 'first' cube select id,name from sample_db_dim order by name @@ -55,7 +55,7 @@ cube select id,name from sample_db_dim where name != 'first' order by name desc cube select count(id) from sample_db_dim cube select count(id) from sample_db_dim group by name cube select count(distinct id) from sample_db_dim -select * from (cube select sample_dim.name name1, sample_dim2.name name2 from sample_dim where sample_dim.name != 'first') a +select * from (cube select sample_dim.name name1, sample_dim2_chain.name name2 from sample_dim where sample_dim.name !='first') a drop table temp1 create table temp1 as cube select id,name from sample_dim select * from temp1 http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/sales-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml index 7ec2ec7..e9e48ce 100644 --- a/lens-examples/src/main/resources/sales-cube.xml +++ b/lens-examples/src/main/resources/sales-cube.xml @@ -56,17 +56,13 @@ <dim_attribute name="delivery_city_id" type="INT" /> <dim_attribute name="customer_city_name" type="string" description="City name to which the customer belongs" display_string="Customer City"> - <ref_spec> - <chain_ref_column chain_name="customer_city" ref_col="name" /> - </ref_spec> + <chain_ref_column chain_name="customer_city" ref_col="name" /> </dim_attribute> <dim_attribute name="production_location"> <hierarchy> <dim_attribute name="production_city_name" type="STRING" description="City name in which the product was produced" display_string="Production City"> - <ref_spec> - <chain_ref_column chain_name="production_city" ref_col="name" /> - </ref_spec> + <chain_ref_column chain_name="production_city" ref_col="name" /> </dim_attribute> <dim_attribute name="production_state" type="STRING" description="State name in which the product was produced" display_string="Production State"/> @@ -76,9 +72,7 @@ </dim_attribute> <dim_attribute name="delivery_city_name" type="STRING" description="City name to which the product was delivered" display_string="Delivery City"> - <ref_spec> - <chain_ref_column chain_name="delivery_city" ref_col="name" /> - </ref_spec> + <chain_ref_column chain_name="delivery_city" ref_col="name" /> </dim_attribute> </dim_attributes> <expressions> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/sample-cube.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml index 7b784ea..3b5abea 100644 --- a/lens-examples/src/main/resources/sample-cube.xml +++ b/lens-examples/src/main/resources/sample-cube.xml @@ -33,12 +33,7 @@ <dim_attributes> <dim_attribute name="dim1" type="INT"/> <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00'/> - <dim_attribute name="dim3" type="INT" join_key="true"> - <ref_spec> - <table_references> - <table_reference table="sample_dim" column="id"/> - </table_references> - </ref_spec> + <dim_attribute name="dim3" type="INT"> </dim_attribute> </dim_attributes> <expressions> @@ -47,4 +42,18 @@ <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/> </expression> </expressions> + <join_chains> + <join_chain name="sample_dim_chain"> + <paths> + <path> + <edges> + <edge> + <from table="sample_cube" column="dim3" /> + <to table="sample_dim" column="id" /> + </edge> + </edges> + </path> + </paths> + </join_chain> + </join_chains> </x_base_cube> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/sample-db-only-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-db-only-dimension.xml b/lens-examples/src/main/resources/sample-db-only-dimension.xml index 4c6bec6..2e8aa64 100644 --- a/lens-examples/src/main/resources/sample-db-only-dimension.xml +++ b/lens-examples/src/main/resources/sample-db-only-dimension.xml @@ -25,14 +25,24 @@ <dim_attribute name="id" type="INT"/> <dim_attribute name="name" type="STRING"/> <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00'/> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00' join_key="true"> - <ref_spec> - <table_references> - <table_reference table="sample_dim2" column="id"/> - </table_references> - </ref_spec> - </dim_attribute> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'/> </attributes> + + <join_chains> + <join_chain name="sample_dim2_chain"> + <paths> + <path> + <edges> + <edge> + <from table="sample_db_dim" column="d2id" /> + <to table="sample_dim2" column="id" /> + </edge> + </edges> + </path> + </paths> + </join_chain> + </join_chains> + <properties> <property name="dimension.sample_db_dim.timed.dimension" value="dt"/> </properties> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-examples/src/main/resources/sample-dimension.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/sample-dimension.xml b/lens-examples/src/main/resources/sample-dimension.xml index 9b97da7..3c2589c 100644 --- a/lens-examples/src/main/resources/sample-dimension.xml +++ b/lens-examples/src/main/resources/sample-dimension.xml @@ -25,14 +25,24 @@ <dim_attribute name="id" type="INT"/> <dim_attribute name="name" type="STRING"/> <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00'/> - <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00' join_key="true"> - <ref_spec> - <table_references> - <table_reference table="sample_dim2" column="id"/> - </table_references> - </ref_spec> - </dim_attribute> + <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'/> </attributes> + + <join_chains> + <join_chain name="sample_dim2_chain"> + <paths> + <path> + <edges> + <edge> + <from table="sample_dim" column="d2id" /> + <to table="sample_dim2" column="id" /> + </edge> + </edges> + </path> + </paths> + </join_chain> + </join_chains> + <properties> <property name="dimension.sample_dim.timed.dimension" value="dt"/> </properties> http://git-wip-us.apache.org/repos/asf/lens/blob/908530f5/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java index 817c84c..70323d3 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java @@ -30,7 +30,7 @@ import javax.xml.datatype.XMLGregorianCalendar; import org.apache.lens.api.metastore.*; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; -import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol; +import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol; import org.apache.lens.server.api.error.LensException; import org.apache.hadoop.hive.metastore.TableType; @@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.InputFormat; import com.google.common.base.Optional; import com.google.common.collect.Maps; + import lombok.extern.slf4j.Slf4j; /** @@ -159,7 +160,7 @@ public final class JAXBUtils { * @param xd * @return {@link org.apache.lens.cube.metadata.CubeDimAttribute} */ - public static CubeDimAttribute hiveDimAttrFromXDimAttr(XDimAttribute xd) { + public static CubeDimAttribute hiveDimAttrFromXDimAttr(XDimAttribute xd) throws LensException { Date startDate = getDateFromXML(xd.getStartTime()); Date endDate = getDateFromXML(xd.getEndTime()); @@ -171,33 +172,12 @@ public final class JAXBUtils { hierarchy.add(hiveDimAttrFromXDimAttr(hd)); } hiveDim = new HierarchicalDimAttribute(xd.getName(), xd.getDescription(), hierarchy); - } else if (xd.getRefSpec() != null && xd.getRefSpec().getTableReferences() != null - && !xd.getRefSpec().getTableReferences().getTableReference().isEmpty()) { - - List<TableReference> dimRefs = new ArrayList<TableReference>( - xd.getRefSpec().getTableReferences().getTableReference().size()); - - for (XTableReference xRef : xd.getRefSpec().getTableReferences().getTableReference()) { - dimRefs.add(new TableReference(xRef.getTable(), xRef.getColumn(), xRef.isMapsToMany())); - } - - hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), - xd.getDescription()), - xd.getDisplayString(), - dimRefs, - startDate, - endDate, - null, - xd.isJoinKey(), - xd.getNumDistinctValues(), - xd.getValues() - ); - } else if (xd.getRefSpec() != null && xd.getRefSpec().getChainRefColumn() != null - && !xd.getRefSpec().getChainRefColumn().isEmpty()) { - hiveDim = new ReferencedDimAtrribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), + } else if (xd.getChainRefColumn() != null + && !xd.getChainRefColumn().isEmpty()) { + hiveDim = new ReferencedDimAttribute(new FieldSchema(xd.getName(), xd.getType().toLowerCase(), xd.getDescription()), xd.getDisplayString(), - getChainRefColumns(xd.getRefSpec().getChainRefColumn()), + getChainRefColumns(xd.getChainRefColumn()), startDate, endDate, null, @@ -334,9 +314,8 @@ public final class JAXBUtils { xd.setDisplayString(cd.getDisplayString()); xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime())); xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime())); - if (cd instanceof ReferencedDimAtrribute) { - ReferencedDimAtrribute rd = (ReferencedDimAtrribute) cd; - XDimAttribute.RefSpec refspec = XCF.createXDimAttributeRefSpec(); + if (cd instanceof ReferencedDimAttribute) { + ReferencedDimAttribute rd = (ReferencedDimAttribute) cd; if (!rd.getChainRefColumns().isEmpty()) { for (ChainRefCol crCol : rd.getChainRefColumns()) { XChainColumn xcc = new XChainColumn(); @@ -347,16 +326,9 @@ public final class JAXBUtils { } else { xcc.setDestTable(baseTable.getChainByName(crCol.getChainName()).getDestTable()); } - refspec.getChainRefColumn().add(xcc); + xd.getChainRefColumn().add(xcc); } - xd.setJoinKey(false); - } else { - List<TableReference> dimRefs = rd.getReferences(); - refspec.setTableReferences(new XTableReferences()); - refspec.getTableReferences().getTableReference().addAll(xTabReferencesFromHiveTabReferences(dimRefs)); - xd.setJoinKey(rd.useAsJoinKey()); - } - xd.setRefSpec(refspec); + } xd.setType(rd.getType()); Optional<Long> numOfDistinctValues = rd.getNumOfDistinctValues(); if (numOfDistinctValues.isPresent()) { @@ -663,7 +635,7 @@ public final class JAXBUtils { return null; } - public static CubeDimensionTable cubeDimTableFromDimTable(XDimensionTable dimensionTable) { + public static CubeDimensionTable cubeDimTableFromDimTable(XDimensionTable dimensionTable) throws LensException { return new CubeDimensionTable(dimensionTable.getDimensionName(), dimensionTable.getTableName(), @@ -673,7 +645,7 @@ public final class JAXBUtils { mapFromXProperties(dimensionTable.getProperties())); } - public static CubeFactTable cubeFactFromFactTable(XFactTable fact) { + public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensException { List<FieldSchema> columns = fieldSchemaListFromColumns(fact.getColumns()); Map<String, Set<UpdatePeriod>> storageUpdatePeriods = getFactUpdatePeriodsFromStorageTables(
