Repository: lens Updated Branches: refs/heads/master 4e81ef4dd -> 08135aa69
LENS-392 : Look ahead timerange should not look for all finer granularity partitions to be present Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08135aa6 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08135aa6 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08135aa6 Branch: refs/heads/master Commit: 08135aa693658aa454e55a2c1e81c9e0fa19444b Parents: 4e81ef4 Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Sep 16 09:50:55 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Sep 16 09:50:55 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/FactPartition.java | 14 ++- .../lens/cube/metadata/TimePartition.java | 9 ++ .../lens/cube/metadata/TimePartitionRange.java | 5 + .../timeline/RangesPartitionTimeline.java | 4 +- .../cube/parse/AbridgedTimeRangeWriter.java | 5 +- .../lens/cube/parse/StorageTableResolver.java | 43 ++++----- .../apache/lens/cube/parse/CubeTestSetup.java | 52 ++++++----- .../lens/cube/parse/TestCubeRewriter.java | 97 ++++++++++---------- 8 files changed, 130 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java index fc2d85b..f934ad3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java @@ -42,6 +42,7 @@ public class FactPartition implements Comparable<FactPartition> { @Getter @Setter private FactPartition containingPart; + @Getter private final DateFormat partFormat; @Getter @Setter @@ -59,7 +60,18 @@ public class FactPartition implements Comparable<FactPartition> { public FactPartition(String partCol, Date partSpec, UpdatePeriod period, FactPartition containingPart, DateFormat partFormat, Set<String> storageTables) { this(partCol, partSpec, period, containingPart, partFormat); - this.storageTables.addAll(storageTables); + if (storageTables != null) { + this.storageTables.addAll(storageTables); + } + } + + public FactPartition(String partCol, TimePartition timePartition) { + this(partCol, timePartition, null, null); + } + + public FactPartition(String partCol, TimePartition timePartition, FactPartition containingPart, Set<String> + storageTables) { + this(partCol, timePartition.getDate(), timePartition.getUpdatePeriod(), containingPart, null, storageTables); } public boolean hasContainingPart() { http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java index d52f168..0026262 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java @@ -121,6 +121,7 @@ public class TimePartition implements Comparable<TimePartition>, Named { return rangeUpto(next()); } + @Override public String getName() { return getDateString(); @@ -129,4 +130,12 @@ public class TimePartition implements Comparable<TimePartition>, Named { public TimePartitionRange emptyRange() throws LensException { return this.rangeUpto(this); } + + public static TimePartition max(TimePartition p1, TimePartition p2) { + return p1.compareTo(p2) >= 0 ? p1 : p2; + } + + public static TimePartition min(TimePartition p1, TimePartition p2) { + return p1.compareTo(p2) < 0 ? p1 : p2; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java index f5f8d4c..01069a5 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.metadata; +import java.util.Date; import java.util.Iterator; import org.apache.lens.cube.parse.DateUtil; @@ -33,6 +34,10 @@ public class TimePartitionRange implements Iterable<TimePartition>, Named { private TimePartition begin; private TimePartition end; + public static TimePartitionRange between(Date from, Date to, UpdatePeriod period) throws LensException { + return TimePartition.of(period, from).rangeUpto(TimePartition.of(period, to)); + } + public TimePartitionRange(TimePartition begin, TimePartition end) throws LensException { if (end.before(begin)) { throw new LensException("condition of creation of timepartition failed: end>=begin"); http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java index 1b9a44a..6c9eb7a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java @@ -132,9 +132,9 @@ public class RangesPartitionTimeline extends PartitionTimeline { private void mergeRanges() { for (int i = 0; i < ranges.size() - 1; i++) { - if (ranges.get(i).getEnd().equals(ranges.get(i + 1).getBegin())) { + if (ranges.get(i).getEnd().compareTo(ranges.get(i + 1).getBegin()) >= 0) { TimePartitionRange removed = ranges.remove(i + 1); - ranges.get(i).setEnd(removed.getEnd()); + ranges.get(i).setEnd(TimePartition.max(removed.getEnd(), ranges.get(i).getEnd())); i--; // check again at same index } else if (ranges.get(i).isEmpty()) { ranges.remove(i); http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java index 2caea56..8681e90 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java @@ -121,11 +121,12 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter { Map<FactPartition, Set<FactPartition>> partitionSetMap = new HashMap<FactPartition, Set<FactPartition>>(); for (FactPartition part : parts) { FactPartition key = part.getContainingPart(); - part.setContainingPart(null); + FactPartition part2 = new FactPartition(part.getPartCol(), part.getPartSpec(), part.getPeriod(), null, part + .getPartFormat(), part.getStorageTables()); if (partitionSetMap.get(key) == null) { partitionSetMap.put(key, Sets.<FactPartition>newTreeSet()); } - partitionSetMap.get(key).add(part); + partitionSetMap.get(key).add(part2); } Map<Set<FactPartition>, Set<FactPartition>> setSetOppositeMap = Maps.newHashMap(); for (Map.Entry<FactPartition, Set<FactPartition>> entry : partitionSetMap.entrySet()) { http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 58d0fa7..68ab5ab 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -369,7 +369,7 @@ class StorageTableResolver implements ContextRewriter { // If no partitions were found, then we'll fallback. String partCol = range.getPartitionColumn(); boolean partColNotSupported = rangeParts.isEmpty(); - for(String storage: cfact.fact.getStorages()) { + for (String storage : cfact.fact.getStorages()) { String storageTableName = getFactOrDimtableStorageTableName(cfact.fact.getName(), storage).toLowerCase(); partColNotSupported &= skipStorageCauses.containsKey(storageTableName) && skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST) @@ -484,11 +484,11 @@ class StorageTableResolver implements ContextRewriter { minimalPartsCopy.retainAll(rangeParts); if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) { rangeToWhere.put( - rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), - minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table); + rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), + minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table); } else { rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql, - cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table); + cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table); } } cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere); @@ -506,7 +506,7 @@ class StorageTableResolver implements ContextRewriter { HashMap<String, SkipStorageCause> skipStorageCauses, PartitionRangesForPartitionColumns missingPartitions) throws LensException { try { - return getPartitions(fact, range, getValidUpdatePeriods(fact), true, skipStorageCauses, + return getPartitions(fact, range, getValidUpdatePeriods(fact), true, failOnPartialData, skipStorageCauses, missingPartitions); } catch (Exception e) { throw new LensException(e); @@ -514,22 +514,22 @@ class StorageTableResolver implements ContextRewriter { } private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range, TreeSet<UpdatePeriod> updatePeriods, - boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, + boolean addNonExistingParts, boolean failOnPartialData, Map<String, SkipStorageCause> skipStorageCauses, PartitionRangesForPartitionColumns missingPartitions) throws Exception { - Set<FactPartition> partitions = new TreeSet<FactPartition>(); + Set<FactPartition> partitions = new TreeSet<>(); if (range != null && range.isCoverableBy(updatePeriods) && getPartitions(fact, range.getFromDate(), range.getToDate(), range.getPartitionColumn(), partitions, - updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions)) { + updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions)) { return partitions; } else { - return new TreeSet<FactPartition>(); + return new TreeSet<>(); } } private boolean getPartitions(CubeFactTable fact, Date fromDate, Date toDate, String partCol, Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods, - boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses, + boolean addNonExistingParts, boolean failOnPartialData, Map<String, SkipStorageCause> skipStorageCauses, PartitionRangesForPartitionColumns missingPartitions) throws Exception { log.info("getPartitions for {} from fromDate:{} toDate:{}", fact, fromDate, toDate); @@ -629,21 +629,18 @@ class StorageTableResolver implements ContextRewriter { log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt); Set<FactPartition> processTimeParts = getPartitions(fact, TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn( - processTimePartCol).build(), newset, false, skipStorageCauses, missingPartitions); + processTimePartCol).build(), newset, true, false, skipStorageCauses, missingPartitions); log.debug("Look ahead partitions: {}", processTimeParts); TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build(); for (FactPartition pPart : processTimeParts) { log.debug("Looking for finer partitions in pPart: {}", pPart); for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { - partitions.add(new FactPartition(partCol, date, pPart.getPeriod(), pPart, - partWhereClauseFormat)); + FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, + partWhereClauseFormat); + updateFactPartitionStorageTablesFrom(fact, innerPart, pPart.getStorageTables()); + partitions.add(innerPart); } log.debug("added all sub partitions blindly in pPart: {}", pPart); - // if (!getPartitions(fact, dt, cal.getTime(), partCol, pPart, partitions, - // newset, false, - // skipStorageCauses, nonExistingParts)) { - // log.info("No partitions found in look ahead range"); - // } } } } @@ -654,7 +651,7 @@ class StorageTableResolver implements ContextRewriter { TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>(); newset.addAll(updatePeriods); newset.remove(interval); - if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, skipStorageCauses, + if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, failOnPartialData, skipStorageCauses, missingPartitions)) { log.debug("Adding non existing partition {}", part); @@ -676,13 +673,13 @@ class StorageTableResolver implements ContextRewriter { } } return getPartitions(fact, fromDate, ceilFromDate, partCol, partitions, - updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions) + updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions) && getPartitions(fact, floorToDate, toDate, partCol, partitions, - updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions); + updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions); } - private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, FactPartition part, - Set<String> storageTableNames) throws LensException, HiveException, ParseException { + void updateFactPartitionStorageTablesFrom(CubeFactTable fact, + FactPartition part, Set<String> storageTableNames) throws LensException, HiveException, ParseException { for (String storageTableName : storageTableNames) { if (client.factPartitionExists(fact, part, storageTableName)) { part.getStorageTables().add(storageTableName); http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 7f56292..67f7ab9 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -19,6 +19,10 @@ package org.apache.lens.cube.parse; +import static java.util.Calendar.DAY_OF_MONTH; +import static java.util.Calendar.HOUR_OF_DAY; +import static java.util.Calendar.MONTH; + import static org.apache.lens.cube.metadata.UpdatePeriod.*; import static org.testng.Assert.assertEquals; @@ -142,31 +146,31 @@ public class CubeTestSetup { log.debug("Test now:{}", NOW); // Figure out if current hour is 0th hour - zerothHour = (cal.get(Calendar.HOUR_OF_DAY) == 0); + zerothHour = (cal.get(HOUR_OF_DAY) == 0); // Figure out last hour - cal.add(Calendar.HOUR_OF_DAY, -1); + cal.add(HOUR_OF_DAY, -1); LAST_HOUR = cal.getTime(); log.debug("LastHour:{}", LAST_HOUR); cal.setTime(NOW); - cal.add(Calendar.DAY_OF_MONTH, -1); + cal.add(DAY_OF_MONTH, -1); ONE_DAY_BACK = cal.getTime(); - cal.add(Calendar.DAY_OF_MONTH, -1); + cal.add(DAY_OF_MONTH, -1); TWODAYS_BACK = cal.getTime(); System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK); // two months back cal.setTime(NOW); - cal.add(Calendar.MONTH, -2); + cal.add(MONTH, -2); TWO_MONTHS_BACK = cal.getTime(); System.out.println("Test TWO_MONTHS_BACK:" + TWO_MONTHS_BACK); // Before 4days cal.setTime(NOW); - cal.add(Calendar.DAY_OF_MONTH, -4); + cal.add(DAY_OF_MONTH, -4); BEFORE_4_DAYS_END = cal.getTime(); - cal.add(Calendar.DAY_OF_MONTH, -2); + cal.add(DAY_OF_MONTH, -2); BEFORE_4_DAYS_START = cal.getTime(); @@ -426,7 +430,7 @@ public class CubeTestSetup { } Calendar cal = new GregorianCalendar(); cal.setTime(dayStart); - if (cal.get(Calendar.DAY_OF_MONTH) != 1) { + if (cal.get(DAY_OF_MONTH) != 1) { addParts(dailyparts, DAILY, dayStart, DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY)); monthStart = DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY); } @@ -478,7 +482,7 @@ public class CubeTestSetup { } Calendar cal = new GregorianCalendar(); cal.setTime(dayStart); - if (cal.get(Calendar.DAY_OF_MONTH) != 1) { + if (cal.get(DAY_OF_MONTH) != 1) { addParts(dailyparts, DAILY, dayStart, DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY)); monthStart = DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY); } @@ -1277,7 +1281,7 @@ public class CubeTestSetup { timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c99); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1290,7 +1294,7 @@ public class CubeTestSetup { timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c99); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } } @@ -1392,7 +1396,7 @@ public class CubeTestSetup { } catch (LensException e) { log.error("Encountered Lens exception.", e); } - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } @@ -1404,7 +1408,7 @@ public class CubeTestSetup { timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c1); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } client.clearHiveTableCache(); @@ -1438,7 +1442,7 @@ public class CubeTestSetup { partitions.add(HOURLY.format().format(temp)); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); storagePartitionDescs.add(sPartSpec); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } client.addPartitions(storagePartitionDescs, c4); @@ -1457,7 +1461,7 @@ public class CubeTestSetup { timeParts.put("ttd2", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c4); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } } @@ -1516,7 +1520,7 @@ public class CubeTestSetup { timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c3); - cal.add(Calendar.HOUR_OF_DAY, 1); + cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } } @@ -2309,10 +2313,10 @@ public class CubeTestSetup { // Add partitions in PIE storage Calendar pcal = Calendar.getInstance(); pcal.setTime(TWODAYS_BACK); - pcal.set(Calendar.HOUR, 0); + pcal.set(HOUR_OF_DAY, 0); Calendar ical = Calendar.getInstance(); ical.setTime(TWODAYS_BACK); - ical.set(Calendar.HOUR, 0); + ical.set(HOUR_OF_DAY, 0); Map<UpdatePeriod, TreeSet<Date>> pTimes = Maps.newHashMap(); pTimes.put(DAILY, Sets.<Date>newTreeSet()); @@ -2343,8 +2347,8 @@ public class CubeTestSetup { pTimes.get(DAILY).add(ptime); iTimes.get(DAILY).add(itime); client.addPartition(sPartSpec, storageName); - pcal.add(Calendar.DAY_OF_MONTH, 1); - ical.add(Calendar.HOUR_OF_DAY, 20); + pcal.add(DAY_OF_MONTH, 1); + ical.add(HOUR_OF_DAY, 20); } else if (p == 2) { // day2 // pt=day2-hour[0-3] it = day1-hour[20-23] // pt=day2 and it=day1 @@ -2372,8 +2376,8 @@ public class CubeTestSetup { pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); client.addPartition(sPartSpec, storageName); - pcal.add(Calendar.HOUR_OF_DAY, 1); - ical.add(Calendar.HOUR_OF_DAY, 1); + pcal.add(HOUR_OF_DAY, 1); + ical.add(HOUR_OF_DAY, 1); } // pt=day2 and it=day2 sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY); @@ -2394,8 +2398,8 @@ public class CubeTestSetup { pTimes.get(HOURLY).add(ptime); iTimes.get(HOURLY).add(itime); client.addPartition(sPartSpec, storageName); - pcal.add(Calendar.HOUR_OF_DAY, 1); - ical.add(Calendar.HOUR_OF_DAY, 1); + pcal.add(HOUR_OF_DAY, 1); + ical.add(HOUR_OF_DAY, 1); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 7e5184c..a58f5fe 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -19,6 +19,8 @@ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY; +import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; import static org.apache.lens.cube.parse.CubeTestSetup.*; @@ -46,6 +48,8 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -91,14 +95,14 @@ public class TestCubeRewriter extends TestQueryRewrite { String qFrom = qFmt.format(from2DaysBackDate); CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr15) from testCube where" - + " time_range_in(d_time, '"+ qFrom + "', '" + qTo + "')", conf); + + " time_range_in(d_time, '" + qFrom + "', '" + qTo + "')", conf); DateFormat fmt = UpdatePeriod.CONTINUOUS.format(); String to = fmt.format(toDate); String from = fmt.format(from2DaysBackDate); String expected = "select SUM((testCube.msr15)) from TestQueryRewrite.c0_testFact_CONTINUOUS testcube" - + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; + + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); System.out.println("expected " + expected); compareQueries(expected, rewrittenQuery.toHQL()); @@ -107,7 +111,7 @@ public class TestCubeRewriter extends TestQueryRewrite { rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where" + " time_range_in(d_time, '" + qFrom + "', '" + qTo + "')", conf); expected = "select SUM((testCube.msr2)) from TestQueryRewrite.c0_testFact testcube" - + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; + + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))"; System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL()); System.out.println("expected " + expected); compareQueries(expected, rewrittenQuery.toHQL()); @@ -117,7 +121,7 @@ public class TestCubeRewriter extends TestQueryRewrite { Date from4DaysBackDate = qCal.getTime(); String qFrom4DaysBackDate = qFmt.format(from4DaysBackDate); LensException th = getLensExceptionInRewrite("select SUM(msr15) from testCube where" - + " time_range_in(d_time, '"+ qFrom4DaysBackDate + "', '" + qTo + "')", getConf()); + + " time_range_in(d_time, '" + qFrom4DaysBackDate + "', '" + qTo + "')", getConf()); assertEquals(th.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode()); } @@ -427,9 +431,9 @@ public class TestCubeRewriter extends TestQueryRewrite { System.out.println("HQL: " + hqlQuery); String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact")); + getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact")); String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact")); + getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact")); System.out.println("Expected1 : " + expected1); System.out.println("Expected2 : " + expected2); @@ -460,9 +464,9 @@ public class TestCubeRewriter extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForDailyAndHourly2days(cubeName, "c1_testfact")); + getWhereForDailyAndHourly2days(cubeName, "c1_testfact")); String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForDailyAndHourly2days(cubeName, "c2_testfact")); + getWhereForDailyAndHourly2days(cubeName, "c2_testfact")); System.out.println("Expected1 : " + expected1); System.out.println("Expected2 : " + expected2); @@ -496,11 +500,11 @@ public class TestCubeRewriter extends TestQueryRewrite { System.out.println("HQL:" + hqlQuery); String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact")); + getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact")); String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact")); + getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact")); String expected3 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, - getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c3_testfact")); + getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c3_testfact")); System.out.println("Expected1 : " + expected1); System.out.println("Expected2 : " + expected2); @@ -1325,7 +1329,7 @@ public class TestCubeRewriter extends TestQueryRewrite { } @Test - public void testFactsWithTimedDimensionWithProcessTimeCol() throws Exception { + public void testLookAhead() throws Exception { String twoDaysITRange = "time_range_in(it, '" + CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','" + CubeTestSetup.getDateUptoHours(NOW) + "')"; @@ -1333,41 +1337,40 @@ public class TestCubeRewriter extends TestQueryRewrite { Configuration conf = getConf(); conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt"); conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); - String hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - String expected = getExpectedQuery(cubeName, "select testcube.dim1, max(testcube.msr3), sum(testcube.msr2) FROM ", - null, " GROUP BY ( testcube . dim1 )", - getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1"), - null); - // TODO compare queries - // compareQueries(expected, hqlQuery); - hqlQuery = - rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - // TODO compare queries - // compareQueries(expected, hqlQuery); - hqlQuery = - rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where " - + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - // TODO compare queries - // compareQueries(expected, hqlQuery); - conf.setInt(CubeQueryConfUtil.getLookAheadPTPartsKey(UpdatePeriod.DAILY), 3); - hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - // TODO compare queries - // compareQueries(expected, hqlQuery); - hqlQuery = - rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - // TODO compare queries - // compareQueries(expected, hqlQuery); - hqlQuery = - rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where " - + twoDaysITRange, conf); - System.out.println("Query With process time col:" + hqlQuery); - // TODO compare queries - // compareQueries(expected, hqlQuery); + CubeQueryContext ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, + conf); + assertEquals(ctx.candidateFacts.size(), 1); + CandidateFact candidateFact = ctx.candidateFacts.iterator().next(); + Set<FactPartition> partsQueried = new TreeSet<>(candidateFact.getPartsQueried()); + Date ceilDay = DateUtil.getCeilDate(TWODAYS_BACK, DAILY); + Date nextDay = DateUtils.addDays(ceilDay, 1); + Date nextToNextDay = DateUtils.addDays(nextDay, 1); + HashSet<String> storageTables = Sets.newHashSet(); + for (String storageTable : candidateFact.getStorageTables()) { + storageTables.add(storageTable.split("\\.")[1]); + } + TreeSet<FactPartition> expectedPartsQueried = Sets.newTreeSet(); + for (TimePartition p : Iterables.concat( + TimePartition.of(HOURLY, TWODAYS_BACK).rangeUpto(TimePartition.of(HOURLY, ceilDay)), + TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)), + TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) { + FactPartition fp = new FactPartition("it", p, null, storageTables); + expectedPartsQueried.add(fp); + } + for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) { + for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) { + FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables); + FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables); + expectedPartsQueried.add(itPartition); + } + } + assertEquals(partsQueried, expectedPartsQueried); + conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_PT_PARTS_PFX, 3); + ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, + conf); + partsQueried = new TreeSet<>(ctx.candidateFacts.iterator().next().getPartsQueried()); + // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 + assertEquals(partsQueried, expectedPartsQueried); } @Test