Repository: lens Updated Branches: refs/heads/master 2901b4b33 -> 86f251a75
LENS-745 : Fix dropping latest partition from timeline Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/86f251a7 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/86f251a7 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/86f251a7 Branch: refs/heads/master Commit: 86f251a75b3a010a795f29be982ee19d9b2ba780 Parents: 2901b4b Author: Rajat Khandelwal <pro...@apache.org> Authored: Mon Aug 31 10:19:56 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Mon Aug 31 10:19:56 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/TimePartition.java | 2 - .../apache/lens/cube/metadata/UpdatePeriod.java | 57 ++++++-- .../timeline/StoreAllPartitionTimeline.java | 5 +- .../org/apache/lens/cube/parse/DateUtil.java | 77 ++++------ .../org/apache/lens/cube/parse/TimeRange.java | 2 +- .../cube/metadata/TestCubeMetastoreClient.java | 21 +-- .../lens/cube/metadata/TestTimePartition.java | 64 +++++---- .../lens/cube/metadata/UpdatePeriodTest.java | 60 ++++++++ .../timeline/TestPartitionTimelines.java | 143 +++++++++++-------- .../apache/lens/cube/parse/TestDateUtil.java | 4 +- 10 files changed, 258 insertions(+), 177 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java index 11e6ef6..d52f168 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java @@ -37,8 +37,6 @@ public class TimePartition implements Comparable<TimePartition>, Named { private TimePartition(@NonNull UpdatePeriod updatePeriod, @NonNull Date date) { this.updatePeriod = updatePeriod; - Calendar cal = Calendar.getInstance(); - cal.setTime(date); this.date = updatePeriod.truncate(date); this.dateString = updatePeriod.format().format(this.date); } http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java index b4dff46..6c61e94 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java @@ -27,22 +27,28 @@ import java.util.Calendar; import java.util.Comparator; import java.util.Date; +import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.parse.DateUtil; +import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang3.time.DateUtils; +import lombok.Getter; + public enum UpdatePeriod implements Named { - SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"), - MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"), - HOURLY(HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"), - DAILY(DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"), - WEEKLY(WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "yyyy-'W'ww"), - MONTHLY(MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"), - QUARTERLY(MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"), - YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"), - CONTINUOUS(Calendar.SECOND, 1, 1.5f, "yyyy-MM-dd-HH-mm-ss"); + SECONDLY("second", SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"), + MINUTELY("minute", MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"), + HOURLY("hour", HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"), + DAILY("day", DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"), + WEEKLY("week", WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "YYYY-'W'ww"), + MONTHLY("month", MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"), + QUARTERLY("quarter", MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"), + YEARLY("year", YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"), + CONTINUOUS("continuous", Calendar.SECOND, 1, 1.5f, "yyyy-MM-dd-HH-mm-ss"); public static final long MIN_INTERVAL = values()[0].weight(); + @Getter + private String unitName; private final int calendarField; private final long weight; /** @@ -159,7 +165,8 @@ public enum UpdatePeriod implements Named { private static ThreadLocal<DateFormat> quarterlyFormat; private static ThreadLocal<DateFormat> yearlyFormat; - UpdatePeriod(int calendarField, long diff, float normalizationFactor, String format) { + UpdatePeriod(String unitName, int calendarField, long diff, float normalizationFactor, String format) { + this.unitName = unitName; this.calendarField = calendarField; this.weight = diff; this.normalizationFactor = normalizationFactor; @@ -178,6 +185,15 @@ public enum UpdatePeriod implements Named { return DateUtil.getNumberofDaysInMonth(date) * DAILY.weight(); } + public static UpdatePeriod fromUnitName(String unitName) throws LensException { + for (UpdatePeriod up : values()) { + if (up.getUnitName().equals(unitName)) { + return up; + } + } + throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unitName); + } + public DateFormat format() { switch (this) { case CONTINUOUS: @@ -221,21 +237,36 @@ public enum UpdatePeriod implements Named { } public Date truncate(Date date) { - if (this.equals(UpdatePeriod.WEEKLY)) { + switch (this) { + case WEEKLY: Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH); Calendar cal = Calendar.getInstance(); cal.setTime(truncDate); cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek()); return cal.getTime(); - } else if (this.equals(UpdatePeriod.QUARTERLY)) { + case QUARTERLY: Date dt = DateUtils.truncate(date, this.calendarField()); dt.setMonth(dt.getMonth() - dt.getMonth() % 3); return dt; - } else { + default: return DateUtils.truncate(date, this.calendarField()); } } + public Calendar truncate(Calendar calendar) { + Calendar cal = Calendar.getInstance(); + cal.setTime(truncate(calendar.getTime())); + return cal; + } + + public void increment(Calendar calendar, int increment) { + switch (this) { + case QUARTERLY: + increment *= 3; + } + calendar.add(calendarField(), increment); + } + public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> { @Override public int compare(UpdatePeriod o1, UpdatePeriod o2) { http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java index c82da7b..a6cb8da 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java @@ -84,11 +84,10 @@ public class StoreAllPartitionTimeline extends PartitionTimeline { if (partitionsStr == null) { return true; } - boolean ret = true; for (String s : StringUtils.split(partitionsStr, ",")) { - ret &= add(TimePartition.of(getUpdatePeriod(), s)); + add(TimePartition.of(getUpdatePeriod(), s)); } - return ret; + return true; } @Override http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java index 486c6b3..90ea63c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java @@ -50,7 +50,18 @@ public final class DateUtil { * NOW -> new java.util.Date() NOW-7DAY -> a date one week earlier NOW (+-) * <NUM>UNIT or Hardcoded dates in DD-MM-YYYY hh:mm:ss,sss */ - public static final String UNIT = "year|month|week|day|hour|minute|second"; + public static final String UNIT; + + static { + StringBuilder sb = new StringBuilder(); + String sep = ""; + for (UpdatePeriod up : UpdatePeriod.values()) { + sb.append(sep).append(up.getUnitName()); + sep = "|"; + } + UNIT = sb.toString(); + } + public static final String GRANULARITY = "\\.(" + UNIT + ")"; public static final String RELATIVE = "(now){1}(" + GRANULARITY + "){0,1}"; public static final Pattern P_RELATIVE = Pattern.compile(RELATIVE, Pattern.CASE_INSENSITIVE); @@ -127,9 +138,11 @@ public final class DateUtil { return resolveAbsoluteDate(str); } } + public static String relativeToAbsolute(String relative) throws LensException { return relativeToAbsolute(relative, new Date()); } + public static String relativeToAbsolute(String relative, Date now) throws LensException { if (RELDATE_VALIDATOR.matcher(relative).matches()) { return ABSDATE_PARSER.get().format(resolveRelativeDate(relative, now)); @@ -166,24 +179,7 @@ public final class DateUtil { Matcher granularityMatcher = P_UNIT.matcher(nowWithGranularity); if (granularityMatcher.find()) { String unit = granularityMatcher.group().toLowerCase(); - if ("year".equals(unit)) { - calendar = DateUtils.truncate(calendar, YEAR); - } else if ("month".equals(unit)) { - calendar = DateUtils.truncate(calendar, MONTH); - } else if ("week".equals(unit)) { - calendar.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); - calendar = DateUtils.truncate(calendar, DAY_OF_MONTH); - } else if ("day".equals(unit)) { - calendar = DateUtils.truncate(calendar, DAY_OF_MONTH); - } else if ("hour".equals(unit)) { - calendar = DateUtils.truncate(calendar, Calendar.HOUR_OF_DAY); - } else if ("minute".equals(unit)) { - calendar = DateUtils.truncate(calendar, Calendar.MINUTE); - } else if ("second".equals(unit)) { - calendar = DateUtils.truncate(calendar, Calendar.SECOND); - } else { - throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unit); - } + calendar = UpdatePeriod.fromUnitName(granularityMatcher.group().toLowerCase()).truncate(calendar); } } @@ -386,11 +382,9 @@ public final class DateUtil { case CONTINUOUS: return getMilliSecondCoveringInfo(from, to, 1000); case MINUTELY: - return getMilliSecondCoveringInfo(from, to, 1000 * 60); case HOURLY: - return getMilliSecondCoveringInfo(from, to, 1000 * 60 * 60); case DAILY: - return getMilliSecondCoveringInfo(from, to, 1000 * 60 * 60 * 24); + return getMilliSecondCoveringInfo(from, to, interval.weight()); case WEEKLY: return getWeeklyCoveringInfo(from, to); case MONTHLY: @@ -404,7 +398,7 @@ public final class DateUtil { } } - private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, int millisInInterval) { + private static CoveringInfo getMilliSecondCoveringInfo(Date from, Date to, long millisInInterval) { long diff = to.getTime() - from.getTime(); return new CoveringInfo((int) (diff / millisInInterval), diff % millisInInterval == 0); } @@ -419,7 +413,11 @@ public final class DateUtil { } public static int getTimeDiff(Date fromDate, Date toDate, UpdatePeriod updatePeriod) { - return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween(); + if (fromDate.before(toDate)) { + return getCoveringInfo(fromDate, toDate, updatePeriod).getCountBetween(); + } else { + return -getCoveringInfo(toDate, fromDate, updatePeriod).getCountBetween(); + } } @Data @@ -436,11 +434,11 @@ public final class DateUtil { @EqualsAndHashCode static class TimeDiff { int quantity; - int calendarField; + UpdatePeriod updatePeriod; - public TimeDiff(int quantity, int calendarField) { + private TimeDiff(int quantity, UpdatePeriod updatePeriod) { this.quantity = quantity; - this.calendarField = calendarField; + this.updatePeriod = updatePeriod; } static TimeDiff parseFrom(String diffStr) throws LensException { @@ -461,34 +459,17 @@ public final class DateUtil { Matcher unitMatcher = P_UNIT.matcher(diffStr); if (unitMatcher.find()) { - String unit = unitMatcher.group().toLowerCase(); - if ("year".equals(unit)) { - return new TimeDiff(qty, YEAR); - } else if ("month".equals(unit)) { - return new TimeDiff(qty, MONTH); - } else if ("week".equals(unit)) { - return new TimeDiff(7 * qty, DAY_OF_MONTH); - } else if ("day".equals(unit)) { - return new TimeDiff(qty, DAY_OF_MONTH); - } else if ("hour".equals(unit)) { - return new TimeDiff(qty, HOUR_OF_DAY); - } else if ("minute".equals(unit)) { - return new TimeDiff(qty, MINUTE); - } else if ("second".equals(unit)) { - return new TimeDiff(qty, SECOND); - } else { - throw new LensException(LensCubeErrorCode.INVALID_TIME_UNIT.getValue(), unit); - } + return new TimeDiff(qty, UpdatePeriod.fromUnitName(unitMatcher.group().toLowerCase())); } - return new TimeDiff(0, SECOND); + return new TimeDiff(0, UpdatePeriod.CONTINUOUS); } public Date offsetFrom(Date time) { - return DateUtils.add(time, calendarField, quantity); + return DateUtils.add(time, updatePeriod.calendarField(), quantity); } public Date negativeOffsetFrom(Date time) { - return DateUtils.add(time, calendarField, -quantity); + return DateUtils.add(time, updatePeriod.calendarField(), -quantity); } } http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java index 5444e71..b02499e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRange.java @@ -197,7 +197,7 @@ public class TimeRange { @Override public Date next() { Date cur = calendar.getTime(); - calendar.add(updatePeriod.calendarField(), increment); + updatePeriod.increment(calendar, increment); counter++; return cur; } http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index c2980af..7d5ed0f 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -28,6 +28,7 @@ import org.apache.lens.cube.metadata.ReferencedDimAtrribute.ChainRefCol; import org.apache.lens.cube.metadata.timeline.EndsAndHolesPartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline; +import org.apache.lens.cube.metadata.timeline.TestPartitionTimelines; import org.apache.lens.cube.parse.TimeRange; import org.apache.lens.server.api.error.LensException; @@ -1447,26 +1448,12 @@ public class TestCubeMetastoreClient { private void assertSameTimelines(String factName, String[] storages, UpdatePeriod updatePeriod, String[] partCols) throws HiveException, LensException { for (String partCol : partCols) { - PartitionTimeline[] timelines = new PartitionTimeline[storages.length]; + ArrayList<PartitionTimeline> timelines = Lists.newArrayList(); for (int i = 0; i < storages.length; i++) { - timelines[i] = client.partitionTimelineCache.get(factName, storages[i], updatePeriod, partCol); + timelines.add(client.partitionTimelineCache.get(factName, storages[i], updatePeriod, partCol)); } - for (int i = 0; i < timelines.length; i++) { - for (int j = i + 1; j < timelines.length; j++) { - assertSameTimelines(timelines[i], timelines[j]); - } - } - } - } - - private void assertSameTimelines(PartitionTimeline timeline1, PartitionTimeline timeline2) { - Iterator<TimePartition> iter1 = timeline1.iterator(); - Iterator<TimePartition> iter2 = timeline2.iterator(); - while (iter1.hasNext()) { - Assert.assertTrue(iter2.hasNext()); - assertEquals(iter1.next(), iter2.next()); + TestPartitionTimelines.assertSameTimelines(timelines); } - Assert.assertFalse(iter2.hasNext()); } private StoragePartitionDesc getStoragePartSpec(String cubeFactName, UpdatePeriod updatePeriod, http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java index 334ea4f..2cbbd0c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestTimePartition.java @@ -25,44 +25,46 @@ import java.util.Date; import org.apache.lens.server.api.error.LensException; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class TestTimePartition { public static final Date NOW = new Date(); - @Test - public void test() throws LensException { - // Test for all update periods - for (UpdatePeriod up : UpdatePeriod.values()) { - // Normal date object parsable - String nowStr = up.format().format(NOW); + @DataProvider(name = "update-periods") + public Object[][] provideUpdatePeriods() { + return UpdatePeriodTest.provideUpdatePeriods(); + } - // Create partition by date object or it's string representation -- both should be same. - TimePartition nowPartition = TimePartition.of(up, NOW); - TimePartition nowStrPartition = TimePartition.of(up, nowStr); - assertEquals(nowPartition, nowStrPartition); - - // Test next and previous - assertTrue(nowPartition.next().after(nowPartition)); - assertTrue(nowPartition.previous().before(nowPartition)); - - // date parse failures should give lens exception - assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(), - TimePartition.getWrongUpdatePeriodMessage(up, "garbage")); - getLensExceptionFromPartitionParsing(up, (Date) null); - getLensExceptionFromPartitionParsing(up, (String) null); - getLensExceptionFromPartitionParsing(up, ""); - - // parse with other update periods - for (UpdatePeriod up2 : UpdatePeriod.values()) { - // handles the equality case and the case where monthly-quarterly have same format strings. - if (up.formatStr().equals(up2.formatStr())) { - continue; - } - // Parsing a string representation with differnet update period should give lens exception. - assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(), - TimePartition.getWrongUpdatePeriodMessage(up2, nowStr)); + @Test(dataProvider = "update-periods") + public void test(UpdatePeriod up) throws LensException { + // Normal date object parsable + String nowStr = up.format().format(NOW); + // Create partition by date object or it's string representation -- both should be same. + TimePartition nowPartition = TimePartition.of(up, NOW); + TimePartition nowStrPartition = TimePartition.of(up, nowStr); + assertEquals(nowPartition, nowStrPartition); + + // Test next and previous + assertTrue(nowPartition.next().after(nowPartition)); + assertTrue(nowPartition.previous().before(nowPartition)); + + // date parse failures should give lens exception + assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(), + TimePartition.getWrongUpdatePeriodMessage(up, "garbage")); + getLensExceptionFromPartitionParsing(up, (Date) null); + getLensExceptionFromPartitionParsing(up, (String) null); + getLensExceptionFromPartitionParsing(up, ""); + + // parse with other update periods + for (UpdatePeriod up2 : UpdatePeriod.values()) { + // handles the equality case and the case where monthly-quarterly have same format strings. + if (up.formatStr().equals(up2.formatStr())) { + continue; } + // Parsing a string representation with differnet update period should give lens exception. + assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(), + TimePartition.getWrongUpdatePeriodMessage(up2, nowStr)); } } http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java new file mode 100644 index 0000000..b21341d --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/UpdatePeriodTest.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata; + +import static org.testng.Assert.assertEquals; + +import java.util.Date; +import java.util.Random; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +public class UpdatePeriodTest { + public static class RandomDateGenerator { + Random random = new Random(); + long begin = 1377424977000L; + long end = 1598349777000L; + + public Date nextDate() { + return new Date(begin + (random.nextLong() % (end - begin))); + } + } + + @DataProvider(name = "update-periods") + public static Object[][] provideUpdatePeriods() { + UpdatePeriod[] values = UpdatePeriod.values(); + Object[][] ret = new Object[values.length][1]; + for (int i = 0; i < values.length; i++) { + ret[i][0] = values[i]; + } + return ret; + } + + @Test(dataProvider = "update-periods") + public void testFormat(UpdatePeriod period) throws Exception { + RandomDateGenerator randomDateGenerator = new RandomDateGenerator(); + for (int i = 0; i < 5000; i++) { + Date randomDate = randomDateGenerator.nextDate(); + randomDate = period.truncate(randomDate); + assertEquals(randomDate, period.format().parse(period.format().format(randomDate))); + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java index 0268bbf..3f9e125 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java @@ -18,25 +18,28 @@ */ package org.apache.lens.cube.metadata.timeline; +import static org.testng.Assert.*; + +import java.lang.reflect.InvocationTargetException; import java.util.*; import org.apache.lens.cube.metadata.TestTimePartition; import org.apache.lens.cube.metadata.TimePartition; import org.apache.lens.cube.metadata.UpdatePeriod; +import org.apache.lens.cube.metadata.UpdatePeriodTest; import org.apache.lens.server.api.error.LensException; -import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.beust.jcommander.internal.Lists; - +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import lombok.extern.slf4j.Slf4j; @Slf4j public class TestPartitionTimelines { private static final String TABLE_NAME = "storage_fact"; - public static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY; private static final String PART_COL = "pt"; private static final List<Class<? extends PartitionTimeline>> TIMELINE_IMPLEMENTATIONS = Arrays.asList( StoreAllPartitionTimeline.class, @@ -44,46 +47,48 @@ public class TestPartitionTimelines { RangesPartitionTimeline.class ); - @Test - public void testPropertiesContractsForAllSubclasses() throws LensException { - for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) { - testPropertiesContract(clazz); + @DataProvider(name = "update-periods") + public Object[][] provideUpdatePeriods() { + return UpdatePeriodTest.provideUpdatePeriods(); + } + + @DataProvider(name = "update-periods-and-timeline-classes") + public Object[][] provideUpdatePeriodsAndTimelineClasses() { + UpdatePeriod[] values = UpdatePeriod.values(); + Object[][] ret = new Object[values.length * TIMELINE_IMPLEMENTATIONS.size()][2]; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < TIMELINE_IMPLEMENTATIONS.size(); j++) { + ret[TIMELINE_IMPLEMENTATIONS.size() * i + j] = new Object[]{ + values[i], + TIMELINE_IMPLEMENTATIONS.get(j), + }; + } } + return ret; } - @Test - public void testEquivalence() throws LensException { + + @Test(dataProvider = "update-periods") + public void testEquivalence(UpdatePeriod period) throws LensException, InvocationTargetException, + NoSuchMethodException, InstantiationException, IllegalAccessException { + final Random randomGenerator = new Random(); for (int j = 0; j < 10; j++) { - Random randomGenerator = new Random(); List<PartitionTimeline> timelines = Lists.newArrayList(); for (Class<? extends PartitionTimeline> clazz : TIMELINE_IMPLEMENTATIONS) { - timelines.add(getInstance(clazz)); + timelines.add(getInstance(clazz, period)); } final List<TimePartition> addedPartitions = Lists.newArrayList(); - for (int i = 0; i < 200; i++) { - int randomInt = randomGenerator.nextInt(100) - 50; - TimePartition part = TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD, + for (int i = 0; i < 20; i++) { + int randomInt = randomGenerator.nextInt(10) - 5; + TimePartition part = TimePartition.of(period, TestTimePartition.timeAtDiff(TestTimePartition.NOW, period, randomInt)); addedPartitions.add(part); for (PartitionTimeline timeline : timelines) { timeline.add(part); } + assertSameTimelines(timelines); } - Iterator<TimePartition> sourceOfTruth = timelines.get(0).iterator(); - List<Iterator<TimePartition>> otherIterators = Lists.newArrayList(); - for (int i = 1; i < TIMELINE_IMPLEMENTATIONS.size() - 1; i++) { - otherIterators.add(timelines.get(i).iterator()); - } - while (sourceOfTruth.hasNext()) { - TimePartition cur = sourceOfTruth.next(); - for (Iterator<TimePartition> iterator : otherIterators) { - Assert.assertTrue(iterator.hasNext()); - Assert.assertEquals(iterator.next(), cur); - } - } - for (Iterator<TimePartition> iterator : otherIterators) { - Assert.assertFalse(iterator.hasNext()); - } + assertSameTimelines(timelines); Collections.shuffle(addedPartitions); Iterator<TimePartition> iter = addedPartitions.iterator(); while (iter.hasNext()) { @@ -92,62 +97,80 @@ public class TestPartitionTimelines { if (!addedPartitions.contains(part)) { for (PartitionTimeline timeline : timelines) { timeline.drop(part); + assertTrue(timeline.isConsistent()); } } } for (PartitionTimeline timeline : timelines) { - Assert.assertTrue(timeline.isEmpty()); + assertTrue(timeline.isEmpty()); } } } + public static void assertSameTimelines(List<PartitionTimeline> timelines) { + List<Iterator<TimePartition>> iterators = Lists.newArrayList(); + for (PartitionTimeline timeline : timelines) { + iterators.add(timeline.iterator()); + } - private <T extends PartitionTimeline> T getInstance(Class<T> clz) { - try { - return clz.getConstructor(String.class, UpdatePeriod.class, String.class) - .newInstance(TABLE_NAME, PERIOD, PART_COL); - } catch (Exception e) { - log.error("Error while getting instance.", e); + while (iterators.get(0).hasNext()) { + Map<Class, TimePartition> parts = Maps.newHashMap(); + for (Iterator<TimePartition> iterator : iterators) { + assertTrue(iterator.hasNext()); + parts.put(iterator.getClass(), iterator.next()); + } + assertEquals(new HashSet<>(parts.values()).size(), 1, "More than one values for next: " + parts.values()); } - return null; + for (Iterator<TimePartition> iterator : iterators) { + assertFalse(iterator.hasNext()); + } + } + + + private <T extends PartitionTimeline> T getInstance(Class<T> clz, UpdatePeriod period) throws + NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + return clz.getConstructor(String.class, UpdatePeriod.class, String.class) + .newInstance(TABLE_NAME, period, PART_COL); } - private <T extends PartitionTimeline> void testPropertiesContract(Class<T> clz) throws LensException { + @Test(dataProvider = "update-periods-and-timeline-classes") + public <T extends PartitionTimeline> void testPropertiesContract(UpdatePeriod period, Class<T> clz) throws + LensException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { // Make two instances, one to modify, other to validate against - T inst1 = getInstance(clz); - T inst2 = getInstance(clz); + T inst1 = getInstance(clz, period); + T inst2 = getInstance(clz, period); // whenever we'll init from props, timeline should become empty. Map<String, String> props = inst1.toProperties(); - Assert.assertTrue(inst2.initFromProperties(props)); + assertTrue(inst2.initFromProperties(props)); // init from props of an empty timeline: should succeed and make the timeline empty - Assert.assertEquals(inst1, inst2); - Assert.assertTrue(inst1.isEmpty()); - Assert.assertTrue(inst2.isEmpty()); + assertEquals(inst1, inst2); + assertTrue(inst1.isEmpty()); + assertTrue(inst2.isEmpty()); // Add single partition and test for non-equivalence - Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.NOW))); - Assert.assertFalse(inst1.equals(inst2)); + assertTrue(inst1.add(TimePartition.of(period, TestTimePartition.NOW))); + assertFalse(inst1.equals(inst2)); // add same parittion in other timeline, test for equality - Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, TestTimePartition.NOW))); - Assert.assertTrue(inst1.isConsistent()); - Assert.assertTrue(inst2.isConsistent()); - Assert.assertEquals(inst1, inst2); + assertTrue(inst2.add(TimePartition.of(period, TestTimePartition.NOW))); + assertTrue(inst1.isConsistent()); + assertTrue(inst2.isConsistent()); + assertEquals(inst1, inst2); // init with blank properties. Should become empty - Assert.assertTrue(inst2.initFromProperties(props)); - Assert.assertFalse(inst1.equals(inst2)); + assertTrue(inst2.initFromProperties(props)); + assertFalse(inst1.equals(inst2)); // init from properties of timeline with single partition. - Assert.assertTrue(inst2.initFromProperties(inst1.toProperties())); - Assert.assertEquals(inst1, inst2); + assertTrue(inst2.initFromProperties(inst1.toProperties())); + assertEquals(inst1, inst2); // clear timelines inst1.initFromProperties(props); inst2.initFromProperties(props); // Make sparse partition range in one, init other from its properties. Test equality. - for (int i = 0; i < 5000; i++) { - Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, TestTimePartition.timeAtDiff(TestTimePartition.NOW, PERIOD, + for (int i = 0; i < 500; i++) { + assertTrue(inst1.add(TimePartition.of(period, TestTimePartition.timeAtDiff(TestTimePartition.NOW, period, i * 2)))); } - Assert.assertTrue(inst1.isConsistent()); + assertTrue(inst1.isConsistent()); inst2.initFromProperties(inst1.toProperties()); - Assert.assertTrue(inst2.isConsistent()); - Assert.assertEquals(inst1, inst2); + assertTrue(inst2.isConsistent()); + assertEquals(inst1, inst2); } } http://git-wip-us.apache.org/repos/asf/lens/blob/86f251a7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java index 89d210f..ab88fbe 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java @@ -262,7 +262,7 @@ public class TestDateUtil { assertEquals(diffs.size(), 1); TimeDiff minusFourDaysDiff = diffs.iterator().next(); assertEquals(minusFourDaysDiff.quantity, -4); - assertEquals(minusFourDaysDiff.calendarField, DAY_OF_MONTH); + assertEquals(minusFourDaysDiff.updatePeriod, DAILY); diffs.clear(); for (String diffStr : plusFourDays) { @@ -271,7 +271,7 @@ public class TestDateUtil { assertEquals(diffs.size(), 1); TimeDiff plusFourDaysDiff = diffs.iterator().next(); assertEquals(plusFourDaysDiff.quantity, 4); - assertEquals(plusFourDaysDiff.calendarField, DAY_OF_MONTH); + assertEquals(plusFourDaysDiff.updatePeriod, DAILY); Date now = new Date(); assertEquals(minusFourDaysDiff.offsetFrom(plusFourDaysDiff.offsetFrom(now)), now); assertEquals(plusFourDaysDiff.offsetFrom(minusFourDaysDiff.offsetFrom(now)), now);