LENS-1466 : Fix for 'Rewritten queries not resolving time_range_in clause'
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/ff4f3726 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/ff4f3726 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/ff4f3726 Branch: refs/heads/master Commit: ff4f3726a25db684ff7623fea68e954712dd925f Parents: 10eef27 Author: Rajitha R <rajitha....@gmail.com> Authored: Fri Sep 1 13:02:32 2017 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Fri Sep 1 15:23:45 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/StorageCandidate.java | 2 +- .../apache/lens/cube/metadata/DateFactory.java | 6 ++ .../apache/lens/cube/parse/CubeTestSetup.java | 47 +++++++++++-- .../lens/cube/parse/TestCubeRewriter.java | 15 ++++- .../resources/schema/cubes/base/testcube.xml | 2 + .../test/resources/schema/facts/summary5.xml | 70 ++++++++++++++++++++ .../src/test/resources/schema/storages/c98.xml | 26 ++++++++ 7 files changed, 159 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 2b1add3..27835b3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -533,9 +533,9 @@ public class StorageCandidate implements Candidate, CandidateTable { log.debug("Adding non existing partition {}", part); if (addNonExistingParts) { // Add non existing partitions for all cases of whether we populate all non existing or not. - this.participatingUpdatePeriods.add(maxInterval); missingPartitions.add(part); if (!failOnPartialData) { + this.participatingUpdatePeriods.add(maxInterval); partitions.add(part); part.getStorageTables().add(storageTable); } http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java index 855f54a..75f2abd 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java @@ -150,6 +150,7 @@ public class DateFactory { // Time Instances as Date Type public static final Date NOW; + public static final Date ONEDAY_BACK; public static final Date TWODAYS_BACK; public static final Date TWO_MONTHS_BACK; public static final Date THIS_MONTH_TRUNCATED; @@ -168,6 +169,7 @@ public class DateFactory { public static final String TWO_DAYS_RANGE_TTD2; public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS; public static final String TWO_DAYS_RANGE_IT; + public static final String ONE_DAY_RANGE_IT; public static final String THIS_YEAR_RANGE; public static final String LAST_YEAR_RANGE; public static final String TWO_MONTHS_RANGE_UPTO_MONTH; @@ -189,6 +191,9 @@ public class DateFactory { // Figure out if current hour is 0th hour zerothHour = getDateStringWithOffset(HOURLY, 0).endsWith("-00"); + ONEDAY_BACK = getDateWithOffset(DAILY, -1); + System.out.println("Test ONEDAY_BACK:" + ONEDAY_BACK); + TWODAYS_BACK = getDateWithOffset(DAILY, -2); System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK); @@ -214,6 +219,7 @@ public class DateFactory { TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY); TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY); TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY); + ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY); THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1); LAST_YEAR_RANGE = getTimeRangeString(YEARLY, -1, 0); TWO_MONTHS_RANGE_UPTO_MONTH = getTimeRangeString(MONTHLY, -2, 0); http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 0366e56..5608b39 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -22,12 +22,7 @@ package org.apache.lens.cube.parse; import static java.util.Calendar.DAY_OF_MONTH; import static java.util.Calendar.HOUR_OF_DAY; -import static org.apache.lens.cube.metadata.DateFactory.BEFORE_4_DAYS; -import static org.apache.lens.cube.metadata.DateFactory.BEFORE_6_DAYS; -import static org.apache.lens.cube.metadata.DateFactory.NOW; -import static org.apache.lens.cube.metadata.DateFactory.TWODAYS_BACK; -import static org.apache.lens.cube.metadata.DateFactory.TWO_MONTHS_BACK; -import static org.apache.lens.cube.metadata.DateFactory.isZerothHour; +import static org.apache.lens.cube.metadata.DateFactory.*; import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY; import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY; import static org.apache.lens.cube.metadata.UpdatePeriod.MINUTELY; @@ -132,6 +127,7 @@ public class CubeTestSetup { private static String c3 = "C3"; private static String c4 = "C4"; private static String c5 = "C5"; + private static String c98 = "C98"; private static String c99 = "C99"; private static Map<String, String> factValidityProperties = Maps.newHashMap(); @Getter @@ -881,10 +877,13 @@ public class CubeTestSetup { CubeFactTable fact3 = client.getCubeFactTable(factName); createPIEParts(client, fact3, c2); - factName = "summary4"; CubeFactTable fact4 = client.getCubeFactTable(factName); createPIEParts(client, fact4, c2); + + factName = "summary5"; + CubeFactTable fact5 = client.getCubeFactTable(factName); + createPIParts(client, fact5, c98); } private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { @@ -994,6 +993,7 @@ public class CubeTestSetup { pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); client.addPartition(sPartSpec, storageName, CubeTableType.FACT); + pcal.add(HOUR_OF_DAY, 1); ical.add(HOUR_OF_DAY, 1); } @@ -1015,6 +1015,39 @@ public class CubeTestSetup { } } + + private void createPIParts(CubeMetastoreClient client, CubeFactTable fact, String storageName) + throws Exception { + // Add partitions in PI storage + //daily partition registered for pt=day1, it = day1 + //hourly partitions registered for pt=day1-hours[0-23] it=day1-hours[0-23] + Calendar pcal = Calendar.getInstance(); + pcal.setTime(ONEDAY_BACK); + pcal.set(HOUR_OF_DAY, 0); + Calendar ical = Calendar.getInstance(); + ical.setTime(ONEDAY_BACK); + ical.set(HOUR_OF_DAY, 0); + + Date ptime = pcal.getTime(); + Date itime = ical.getTime(); + Map<String, Date> timeParts = new HashMap<String, Date>(); + + timeParts.put("pt", ptime); + timeParts.put("it", itime); + StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); + for (int i = 0; i < 24; i++) { + ptime = pcal.getTime(); + itime = ical.getTime(); + timeParts.put("pt", ptime); + timeParts.put("it", itime); + sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); + client.addPartition(sPartSpec, storageName, CubeTableType.FACT); + pcal.add(HOUR_OF_DAY, 1); + ical.add(HOUR_OF_DAY, 1); + } + } + public static void printQueryAST(String query, String label) throws LensException { System.out.println("--" + label + "--AST--"); System.out.println("--query- " + query); http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index bc0fa2e..9b54957 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -65,7 +65,7 @@ public class TestCubeRewriter extends TestQueryRewrite { @BeforeTest public void setupDriver() throws Exception { conf = LensServerAPITestUtil.getConfiguration( - DRIVER_SUPPORTED_STORAGES, "C0,C1,C2", + DRIVER_SUPPORTED_STORAGES, "C0,C1,C2,C98", DISABLE_AUTO_JOINS, true, ENABLE_SELECT_TO_GROUPBY, true, ENABLE_GROUP_BY_TO_SELECT, true, @@ -1470,6 +1470,19 @@ public class TestCubeRewriter extends TestQueryRewrite { } @Test + public void testTimeRangeIn() throws Exception { + //check whether time_range_in is resolving in cube rewrite + Configuration conf = getConf(); + conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt"); + conf.set(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, "true"); + conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); + CubeQueryContext ctx = rewriteCtx("select dim1, sum(msr23)" + " from testCube" + " where " + ONE_DAY_RANGE_IT, + conf); + String rewrittenQuery = ctx.toHQL(); + assertTrue(!rewrittenQuery.contains("time_range_in")); + } + + @Test public void testCubeQueryWithMultipleRanges() throws Exception { String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " OR " http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/cubes/base/testcube.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml index 9d2bb02..4cafd31 100644 --- a/lens-cube/src/test/resources/schema/cubes/base/testcube.xml +++ b/lens-cube/src/test/resources/schema/cubes/base/testcube.xml @@ -32,6 +32,8 @@ <property name="cube.timedim.partition.test_time_dim2" value="ttd2"/> </properties> <measures> + <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr23" display_string="Measure23" + description="twenty third measure"/> <measure _type="FLOAT" default_aggr="SUM" unit="RS" name="msr21" display_string="Measure22" description="second measure"/> <measure _type="BIGINT" default_aggr="COUNT" name="msr4" display_string="Measure4" description="fourth measure"/> http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/facts/summary5.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/facts/summary5.xml b/lens-cube/src/test/resources/schema/facts/summary5.xml new file mode 100644 index 0000000..a4bbf71 --- /dev/null +++ b/lens-cube/src/test/resources/schema/facts/summary5.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_fact_table name="summary5" cube_name="testCube" weight="15.0" xmlns="uri:lens:cube:0.1"> + <columns> + <column name="msr23" _type="float" comment="twenty third measure"/> + <column name="dim1" _type="string" comment="dim1"/> + <column name="dim2" _type="string" comment="dim2"/> + </columns> + <properties> + <property name="cube.fact.relative.start.time" value="now.year - 90 days"/> + <property name="cube.table.summary5.weight" value="15.0"/> + <property name="cube.fact.summary5.storages" value="C98"/> + <property name="cube.fact.summary5.c98.updateperiods" value="HOURLY,DAILY"/> + </properties> + <storage_tables> + <storage_table> + <storage_name>C98</storage_name> + <update_periods> + <update_period_table_descriptor> + <update_period>HOURLY</update_period> + <table_desc external="false"> + <part_cols> + <column name="pt" _type="string" comment="date partition"/> + <column name="it" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.start.times" value="now -4 days"/> + </table_parameters> + <time_part_cols>pt</time_part_cols> + <time_part_cols>it</time_part_cols> + </table_desc> + </update_period_table_descriptor> + <update_period_table_descriptor> + <update_period>DAILY</update_period> + <table_desc external="false"> + <part_cols> + <column name="pt" _type="string" comment="date partition"/> + <column name="it" _type="string" comment="date partition"/> + </part_cols> + <table_parameters> + <property name="cube.storagetable.start.times" value="now.month -3 months"/> + </table_parameters> + <time_part_cols>pt</time_part_cols> + <time_part_cols>it</time_part_cols> + </table_desc> + </update_period_table_descriptor> + </update_periods> + </storage_table> + </storage_tables> +</x_fact_table> + http://git-wip-us.apache.org/repos/asf/lens/blob/ff4f3726/lens-cube/src/test/resources/schema/storages/c98.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/resources/schema/storages/c98.xml b/lens-cube/src/test/resources/schema/storages/c98.xml new file mode 100644 index 0000000..550b421 --- /dev/null +++ b/lens-cube/src/test/resources/schema/storages/c98.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<x_storage name="c98" classname="org.apache.lens.cube.metadata.HDFSStorage" xmlns="uri:lens:cube:0.1"> + <properties> + <property name="cube.table.c98.weight" value="0.0"/> + </properties> +</x_storage> \ No newline at end of file
