Repository: oozie Updated Branches: refs/heads/master 77cd1fe20 -> 91c3d0c02
OOZIE-2726 Flaky test due to daylight saving changes (sasishsaley, andras.piros via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/91c3d0c0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/91c3d0c0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/91c3d0c0 Branch: refs/heads/master Commit: 91c3d0c02bbd4ff7c8c115ecf676fe1559fa6d03 Parents: 77cd1fe Author: Gezapeti Cseh <[email protected]> Authored: Tue Mar 27 10:17:54 2018 +0200 Committer: Gezapeti Cseh <[email protected]> Committed: Tue Mar 27 10:17:54 2018 +0200 ---------------------------------------------------------------------- .../CoordMaterializeTransitionXCommand.java | 19 +- .../command/coord/DaylightOffsetCalculator.java | 92 ++++++ .../TestCoordMaterializeTransitionXCommand.java | 291 +++++++++++-------- .../coord/TestDaylightOffsetCalculator.java | 126 ++++++++ release-log.txt | 1 + 5 files changed, 408 insertions(+), 121 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java index 2b91253..b4e62e9 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java @@ -35,7 +35,6 @@ import org.apache.oozie.command.PreconditionException; import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand; import org.apache.oozie.coord.CoordUtils; import org.apache.oozie.coord.TimeUnit; -import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluatorUtil; import org.apache.oozie.executor.jpa.BatchQueryExecutor; import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry; import org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor; @@ -223,7 +222,6 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo if (endMatdTime.compareTo(jobEndTime) > 0) { endMatdTime = jobEndTime; } - LOG.debug("Materializing coord job id=" + jobId + ", start=" + DateUtils.formatDateOozieTZ(startMatdTime) + ", end=" + DateUtils.formatDateOozieTZ(endMatdTime) + ", window=" + materializationWindow); } @@ -457,6 +455,9 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo } boolean firstMater = true; + + end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end); + while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) { if (pause != null && effStart.compareTo(pause) >= 0) { break; @@ -563,8 +564,12 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo job.setStatus(Job.Status.RUNNING); } job.setPending(); + Calendar end = Calendar.getInstance(); + end.setTime(jobEndTime); + + end = calculateEndTimeWithDSTOffset(end); - if (jobEndTime.compareTo(endMatdTime) <= 0) { + if (end.getTime().compareTo(endMatdTime) <= 0) { LOG.info("[" + job.getId() + "]: all actions have been materialized, set pending to true"); // set doneMaterialization to true when materialization is done job.setDoneMaterialization(); @@ -574,6 +579,14 @@ public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCo job.setNextMaterializedTime(endMatdTime); } + private Calendar calculateEndTimeWithDSTOffset(final Calendar endTime) { + final TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone()); + final Calendar start = Calendar.getInstance(appTz); + start.setTime(startMatdTime); + + return new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, endTime); + } + /* (non-Javadoc) * @see org.apache.oozie.command.XCommand#getKey() */ http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java b/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java new file mode 100644 index 0000000..79afe18 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/command/coord/DaylightOffsetCalculator.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.command.coord; + +import java.util.Calendar; +import java.util.Date; +import java.util.TimeZone; + +/** + * Calculates daylight offset for a given {@code target} {@link Calendar}, given a {@link TimeZone}, and both + * {@link #startMatdTime} and {@link #endMatdTime} {@link Calendar} instances. + */ +class DaylightOffsetCalculator { + private final Date startMatdTime; + private final Date endMatdTime; + + DaylightOffsetCalculator(final Date startMatdTime, final Date endMatdTime) { + this.startMatdTime = startMatdTime; + this.endMatdTime = endMatdTime; + } + + /** + * It adjusts {@code target} according to daylight saving time difference between {@link #startMatdTime} and + * {@link #endMatdTime}. + * <ul> + * <li>if {@link #startMatdTime} is in DST (e.g. PDT) and {@link #endMatdTime} is in non-DST (e.g. PST), then we need to + * add {@link Calendar#DST_OFFSET} to {@code target}</li> + * <li>if {@link #startMatdTime} is in non-DST (e.g. PST) and {@link #endMatdTime} is in DST (e.g. PDT), then we need to + * subtract {@link Calendar#DST_OFFSET} to {@code target} + * <li>otherwise, we do not adjust {@code target} and simply return + * </ul> + * + * @param tz {@link TimeZone} for {@code target} + * @param target the {@link Calendar} to modify + * @return adjusted {@code target} according to DST offset between {@link #startMatdTime} and {@link #endMatdTime} + */ + Calendar calculate(final TimeZone tz, final Calendar target) { + final Calendar targetWithDSTOffset = Calendar.getInstance(); + targetWithDSTOffset.setTimeInMillis(target.getTime().getTime() + getDSTOffset(tz, startMatdTime, endMatdTime)); + + return targetWithDSTOffset; + } + + /** + * Calculates daylight saving time difference between {@code beginDate} and {@code endDate}. + * <ul> + * <li>if {@code beginDate} is in DST (e.g. PDT) and {@code endDate} is in non-DST (e.g. PST), then we need to return + * {@link Calendar#DST_OFFSET}</li> + * <li>if {@code beginDate} is in non-DST (e.g. PST) and {@code endDate} is in DST (e.g. PDT), then we need to return + * {@code -1 *} {@link Calendar#DST_OFFSET}</li> + * <li>otherwise, return {@code 0}</li> + * </ul> + * + * @param tz {@link TimeZone} for {@code beginDate} and {@code endDate} + * @param beginDate the beginning {@link Date} of the reference interval + * @param endDate the ending {@link Date} of the reference interval + * @return DST offset between {@code beginDate} and {@code endDate} + */ + static long getDSTOffset(final TimeZone tz, final Date beginDate, final Date endDate) { + if (tz.inDaylightTime(beginDate) && !tz.inDaylightTime(endDate)) { + final Calendar cal = Calendar.getInstance(tz); + cal.setTime(beginDate); + + return cal.get(Calendar.DST_OFFSET); + } + + if (!tz.inDaylightTime(beginDate) && tz.inDaylightTime(endDate)) { + final Calendar cal = Calendar.getInstance(tz); + cal.setTime(endDate); + + return -cal.get(Calendar.DST_OFFSET); + } + + return 0; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java index 61bbbfe..323d34a 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java @@ -20,6 +20,7 @@ package org.apache.oozie.command.coord; import java.io.File; import java.sql.Timestamp; +import java.text.ParseException; import java.util.Arrays; import java.util.Calendar; import java.util.Date; @@ -75,10 +76,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.parseDateOozieTZ("2009-03-06T010:00Z"); Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordAction(job.getId() + "@1"); } + private int hoursToSeconds(final int hours) { + return new Long(java.util.concurrent.TimeUnit.HOURS.toSeconds(hours)).intValue(); + } + public void testActionMaterForHcatalog() throws Exception { Services.get().destroy(); Services services = super.setupServicesForHCatalog(); @@ -87,7 +92,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z"); CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-hcat.xml", CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); CoordinatorActionBean actionBean = getCoordAction(job.getId() + "@1"); assertEquals("file://dummyhdfs/2009/05/_SUCCESS" + CoordCommandUtils.RESOLVED_UNRESOLVED_SEPARATOR + "${coord:latestRange(-1,0)}", actionBean.getMissingDependencies()); @@ -106,7 +111,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-neg-hcat.xml", CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0); try { - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); fail("Expected Command exception but didn't catch any"); } catch (CommandException e) { @@ -125,7 +130,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-11T10:00Z"); CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-matd-relative.xml", CoordinatorJob.Status.RUNNING, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); } public void testActionMaterWithCronFrequency1() throws Exception { @@ -133,16 +138,17 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "10,20 * * * *"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:20Z")}; - checkCoordActionsNominalTime(job.getId(), 2, nominalTimes); + final int expectedNominalTimeCount = 2; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 2); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:10Z")); } catch (JPAExecutorException se) { @@ -156,7 +162,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "10-20 * * * *"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:11Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:12Z"), @@ -168,13 +174,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { DateUtils.parseDateOozieTZ("2013-07-18T00:18Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:19Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"),}; - checkCoordActionsNominalTime(job.getId(), 11, nominalTimes); + final int expectedNominalTimeCount = 11; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 11); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:10Z")); } catch (JPAExecutorException se) { @@ -188,14 +195,15 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "0/15 2 * 5-7 4,5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); - checkCoordActionsNominalTime(job.getId(), 0, new Date[]{}); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); + final int expectedNominalTimeCount = 0; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, new Date[]{}); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 0); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T02:00Z")); } catch (JPAExecutorException se) { @@ -209,19 +217,19 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "0/15 * * 5-7 4,5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:15Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:30Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:45Z"),}; - checkCoordActionsNominalTime(job.getId(), 4, nominalTimes); - + final int expectedNominalTimeCount = 4; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 4); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:00Z")); } catch (JPAExecutorException se) { @@ -235,17 +243,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "20/15 * * 5-7 4,5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:35Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:50Z"),}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 3); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:20Z")); } catch (JPAExecutorException se) { @@ -259,17 +268,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "20"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:40Z"),}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 3); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:00Z")); } catch (JPAExecutorException se) { @@ -283,17 +293,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "20/15 * * 7,10 THU"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:35Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:50Z"),}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 3); + assertEquals(expectedNominalTimeCount, job.getLastActionNumber()); assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-07-18T01:20Z")); } catch (JPAExecutorException se) { @@ -310,17 +321,18 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setMatThrottling(3); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:10Z"), DateUtils.parseDateOozieTZ("2013-07-18T00:20Z")}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertFalse(job.isDoneMaterialization()); - assertEquals(3, job.getLastActionNumber()); + assertEquals(expectedNominalTimeCount, job.getLastActionNumber()); assertEquals(DateUtils.parseDateOozieTZ("2013-07-18T00:30Z"), job.getNextMaterializedTime()); } catch (JPAExecutorException se) { @@ -342,19 +354,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setTimeUnit(Timeunit.CRON); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); final String startPlusOneHour = "2013-03-10T09:00Z"; final String startPlusTwoHours = "2013-03-10T10:00Z"; final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), DateUtils.parseDateOozieTZ(startPlusOneHour), DateUtils.parseDateOozieTZ(startPlusTwoHours)}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { final JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertFalse("coordinator job shouldn't have yet been materialized", job.isDoneMaterialization()); - assertEquals("coordinator action count mismatch", 3, job.getLastActionNumber()); + assertEquals("coordinator action count mismatch", expectedNominalTimeCount, job.getLastActionNumber()); final String startPlusThreeHours = "2013-03-10T11:00Z"; assertEquals("coordinator next materialization time mismatch", DateUtils.parseDateOozieTZ(startPlusThreeHours), job.getNextMaterializedTime()); @@ -365,33 +378,41 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { } } - public void testCronFrequencyCatchupThrottleEqualsDuration() throws Exception { + public void testCronFrequencyCatchupThrottleEqualsDurationDSTChange() throws Exception { final String startInThePast = "2013-03-10T08:00Z"; - final Date startTime = DateUtils.parseDateOozieTZ(startInThePast); + final Date startTimeBeforeDSTChange = DateUtils.parseDateOozieTZ(startInThePast); final String startPlusTwoHoursAndSome = "2013-03-10T10:01Z"; - final Date endTime = DateUtils.parseDateOozieTZ(startPlusTwoHoursAndSome); - CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - job.setNextMaterializedTime(startTime); + final Date endTimeAfterDSTChange = DateUtils.parseDateOozieTZ(startPlusTwoHoursAndSome); + CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, + startTimeBeforeDSTChange, + endTimeAfterDSTChange, + false, + false, + 0); + job.setNextMaterializedTime(startTimeBeforeDSTChange); job.setMatThrottling(3); final String everyHour = "0 * * * *"; job.setFrequency(everyHour); job.setTimeUnit(Timeunit.CRON); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); + final String startPlusOneHour = "2013-03-10T09:00Z"; - final String startPlusTwoHours = "2013-03-10T10:00Z"; - final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), - DateUtils.parseDateOozieTZ(startPlusOneHour), - DateUtils.parseDateOozieTZ(startPlusTwoHours)}; - checkCoordActionsNominalTime(job.getId(), 3, nominalTimes); + final Date[] nominalTimesWithDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), + DateUtils.parseDateOozieTZ(startPlusOneHour)}; + checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithDSTChange); + + checkTwoActionsAfterCatchup(job); + } + private void checkTwoActionsAfterCatchup(CoordinatorJobBean job) throws ParseException { try { final JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue("coordinator job should have already been materialized", job.isDoneMaterialization()); - assertEquals("coordinator action count mismatch", 3, job.getLastActionNumber()); - final String startPlusThreeHours = "2013-03-10T11:00Z"; + assertEquals("coordinator action count mismatch", 2, job.getLastActionNumber()); + final String startPlusThreeHours = "2013-03-10T10:00Z"; assertEquals("coordinator next materialization time mismatch", DateUtils.parseDateOozieTZ(startPlusThreeHours), job.getNextMaterializedTime()); } @@ -401,7 +422,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { } } - public void testCronFrequencyCatchupThrottleMoreThanDuration() throws Exception { + public void testCronFrequencyCatchupThrottleMoreThanDurationNoDSTChange() throws Exception { final String startInThePast = "2013-03-10T08:00Z"; final Date startTime = DateUtils.parseDateOozieTZ(startInThePast); final String startPlusOneHourAndSome = "2013-03-10T09:01Z"; @@ -414,25 +435,14 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setTimeUnit(Timeunit.CRON); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); + final String startPlusOneHour = "2013-03-10T09:00Z"; - final Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), + final Date[] nominalTimesWithoutDSTChange = new Date[] {DateUtils.parseDateOozieTZ(startInThePast), DateUtils.parseDateOozieTZ(startPlusOneHour)}; - checkCoordActionsNominalTime(job.getId(), 2, nominalTimes); + checkCoordActionsNominalTime(job.getId(), 2, nominalTimesWithoutDSTChange); - try { - final JPAService jpaService = Services.get().get(JPAService.class); - job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); - assertTrue("coordinator job should have already been materialized", job.isDoneMaterialization()); - assertEquals("coordinator action count mismatch", 2, job.getLastActionNumber()); - final String startPlusTwoHours = "2013-03-10T10:00Z"; - assertEquals("coordinator next materialization time mismatch", - DateUtils.parseDateOozieTZ(startPlusTwoHours), job.getNextMaterializedTime()); - } - catch (final JPAExecutorException se) { - se.printStackTrace(); - fail("Job ID " + job.getId() + " was not stored properly in db"); - } + checkTwoActionsAfterCatchup(job); } public void testActionMaterWithDST1() throws Exception { @@ -440,19 +450,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2013-03-10T12:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "0 * * * *"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600*4).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(4)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2013-03-10T08:00Z"), DateUtils.parseDateOozieTZ("2013-03-10T09:00Z"), DateUtils.parseDateOozieTZ("2013-03-10T10:00Z"), - DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"),}; - checkCoordActionsNominalTime(job.getId(), 4, nominalTimes); + }; + final int expectedNominalTimeCount = 3; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 4); - assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2013-03-10T12:00Z")); + assertEquals(expectedNominalTimeCount, job.getLastActionNumber()); + assertEquals(DateUtils.parseDateOozieTZ("2013-03-10T11:00Z"), job.getNextMaterializedTime()); } catch (JPAExecutorException se) { se.printStackTrace(); @@ -465,19 +476,20 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2012-11-04T11:00Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "0 * * * *"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600*4).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(4)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2012-11-04T07:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T08:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T09:00Z"), - DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"),}; - checkCoordActionsNominalTime(job.getId(), 4, nominalTimes); + DateUtils.parseDateOozieTZ("2012-11-04T10:00Z"), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z") }; + final int expectedNominalTimeCount = 5; + checkCoordActionsNominalTime(job.getId(), expectedNominalTimeCount, nominalTimes); try { JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertTrue(job.isDoneMaterialization()); - assertEquals(job.getLastActionNumber(), 4); - assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T11:00Z")); + assertEquals(job.getLastActionNumber(), expectedNominalTimeCount); + assertEquals(job.getNextMaterializedTime(), DateUtils.parseDateOozieTZ("2012-11-04T12:00Z")); } catch (JPAExecutorException se) { se.printStackTrace(); @@ -489,7 +501,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z"); Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T10:04Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2009-03-06T10:00Z")}; checkCoordActionsNominalTime(job.getId(), 1, nominalTimes); } @@ -499,7 +511,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z"); Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T10:08Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); Date[] nominalTimes = new Date[] {DateUtils.parseDateOozieTZ("2009-03-06T10:00Z"), DateUtils.parseDateOozieTZ("2009-03-06T10:05Z")}; checkCoordActionsNominalTime(job.getId(), 2, nominalTimes); @@ -515,10 +527,10 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date endTime = DateUtils.parseDateOozieTZ("2009-03-06T10:14Z"); Date pauseTime = DateUtils.parseDateOozieTZ("2009-03-06T09:58Z"); final CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, "5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); - waitFor(1000*60, new Predicate() { + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); + waitFor(60_000, new Predicate() { public boolean evaluate() throws Exception { - return (getStatus(job.getId()) == CoordinatorJob.Status.PAUSED?true:false); + return (getStatus(job.getId()) == CoordinatorJob.Status.PAUSED); } }); checkCoordActions(job.getId(), 0, CoordinatorJob.Status.PAUSED); @@ -531,7 +543,8 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("5"); job.setTimeUnit(Timeunit.MINUTE); job.setMatThrottling(20); - String dryRunOutput = new CoordMaterializeTransitionXCommand(job, 3600, startTime, endTime).materializeActions(true); + String dryRunOutput = new CoordMaterializeTransitionXCommand(job, hoursToSeconds(1), startTime, endTime) + .materializeActions(true); String[] actions = dryRunOutput.split("action for new instance"); assertEquals(3, actions.length -1); for(int i = 1; i < actions.length; i++) { @@ -545,7 +558,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date pauseTime = null; CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, pauseTime, 300, "5"); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordActionsTimeout(job.getId() + "@1", 300); } @@ -553,7 +566,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z"); Date endTime = DateUtils.parseDateOozieTZ("2009-02-03T23:59Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING); } @@ -561,7 +574,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z"); Date endTime = DateUtils.parseDateOozieTZ("2009-02-03T23:59Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordWaiting(job.getId(), job.getMatThrottling()); } @@ -574,7 +587,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.parseDateOozieTZ("2099-02-01T01:00Z"); Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.PREP); } @@ -587,7 +600,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.toDate(new Timestamp(System.currentTimeMillis() + 180 * 1000)); Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING); } @@ -610,7 +623,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { + "<workflow>" + "<app-path>hdfs:///tmp/workflows/</app-path>" + "</workflow>" + "</action>" + "</coordinator-app>"; CoordinatorJobBean job = addRecordToCoordJobTable(coordXml); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); JPAService jpaService = Services.get().get(JPAService.class); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertEquals(CoordinatorJob.Status.FAILED, job.getStatus()); @@ -629,7 +642,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { Date startTime = DateUtils.toDate(new Timestamp(System.currentTimeMillis() + 360 * 1000)); Date endTime = DateUtils.parseDateOozieTZ("2099-02-03T23:59Z"); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.PREP); } @@ -653,7 +666,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("1"); job.setTimeUnitStr("DAY"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertEquals(new Date(startTime.getTime() + TIME_IN_DAY * 3), job.getNextMaterializedTime()); @@ -666,7 +679,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("1"); job.setTimeUnitStr("HOUR"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); assertEquals(new Date(startTime.getTime() + TIME_IN_HOURS * 10), job.getNextMaterializedTime()); @@ -679,13 +692,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("1"); job.setTimeUnitStr("DAY"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); // If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for // that because startTime and endTime assume GMT Date next = new Date(startTime.getTime() + TIME_IN_DAY * 3); TimeZone tz = TimeZone.getTimeZone(job.getTimeZone()); - next.setTime(next.getTime() - getDSTOffset(tz, startTime, next)); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); assertEquals(next, job.getNextMaterializedTime()); // test with hours, time should not pass the current time. @@ -697,13 +710,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("1"); job.setTimeUnitStr("DAY"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); // If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for // that because startTime and endTime assume GMT next = new Date(startTime.getTime() + TIME_IN_DAY); tz = TimeZone.getTimeZone(job.getTimeZone()); - next.setTime(next.getTime() - getDSTOffset(tz, startTime, next)); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); assertEquals(next, job.getNextMaterializedTime()); // for current job in min, should not exceed hour windows @@ -714,13 +727,13 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("5"); job.setTimeUnitStr("MINUTE"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); // If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for // that because startTime and endTime assume GMT next = new Date(startTime.getTime() + TIME_IN_HOURS); tz = TimeZone.getTimeZone(job.getTimeZone()); - next.setTime(next.getTime() - getDSTOffset(tz, startTime, next)); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); assertEquals(next, job.getNextMaterializedTime()); // for current job in hour, should not exceed hour windows @@ -731,27 +744,83 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setFrequency("1"); job.setTimeUnitStr("DAY"); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); // If the startTime and endTime straddle a DST shift (the Coord is in "America/Los_Angeles"), then we need to adjust for // that because startTime and endTime assume GMT next = new Date(startTime.getTime() + TIME_IN_DAY); tz = TimeZone.getTimeZone(job.getTimeZone()); - next.setTime(next.getTime() - getDSTOffset(tz, startTime, next)); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); assertEquals(next, job.getNextMaterializedTime()); + // Case: job started in Daylight time, and materialization is in + // Standard time + startTime = getDaylightCalendar().getTime(); + endTime = new Date(startTime.getTime() + TIME_IN_DAY * 3); + job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); + job.setNextMaterializedTime(startTime); + job.setMatThrottling(10); + job.setFrequency("1"); + job.setTimeUnitStr("DAY"); + CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); + job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); + // If the startTime and endTime straddle a DST shift (the Coord is in + // "America/Los_Angeles"), then we need to adjust for + // that because startTime and endTime assume GMT + next = new Date(startTime.getTime() + TIME_IN_DAY * 3); + tz = TimeZone.getTimeZone(job.getTimeZone()); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); + + assertEquals(next, job.getNextMaterializedTime()); + + // Case: job started in Standard time, and materialization is in + // Daylight time + Calendar c = getStandardCalendar(); + startTime = c.getTime(); + endTime = new Date(startTime.getTime() + TIME_IN_DAY * 3); + job = addRecordToCoordJobTable(CoordinatorJob.Status.PREP, startTime, endTime, false, false, 0); + job.setNextMaterializedTime(startTime); + job.setMatThrottling(10); + job.setFrequency("1"); + job.setTimeUnitStr("DAY"); + CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); + new CoordMaterializeTransitionXCommand(job, hoursToSeconds(1), startTime, endTime).call(); + job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId())); + // If the startTime and endTime straddle a DST shift (the Coord is in + // "America/Los_Angeles"), then we need to adjust for + // that because startTime and endTime assume GMT + next = new Date(startTime.getTime() + TIME_IN_DAY * 3); + tz = TimeZone.getTimeZone(job.getTimeZone()); + next.setTime(next.getTime() + DaylightOffsetCalculator.getDSTOffset(tz, startTime, next)); + assertEquals(next, job.getNextMaterializedTime()); + + } + + Calendar getDaylightCalendar() { + final Calendar daylight = Calendar.getInstance(); + daylight.set(2012, 10, 2, 15, 28, 00); + + return daylight; + } + + Calendar getStandardCalendar() { + final Calendar standard = Calendar.getInstance(); + standard.set(2013, 2, 9, 15, 28, 00); + + return standard; } public void testLastOnlyMaterialization() throws Exception { long now = System.currentTimeMillis(); - Date startTime = DateUtils.toDate(new Timestamp(now - 180 * 60 * 1000)); // 3 hours ago - Date endTime = DateUtils.toDate(new Timestamp(now + 180 * 60 * 1000)); // 3 hours from now + Date startTime = DateUtils.toDate(new Timestamp(now - 180 * 60 * 1000)); // 3 secondsFromHours ago + Date endTime = DateUtils.toDate(new Timestamp(now + 180 * 60 * 1000)); // 3 secondsFromHours from now CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, -1, "10", CoordinatorJob.Execution.LAST_ONLY); // This would normally materialize the throttle amount and within a 1 hour window; however, with LAST_ONLY this should // ignore those parameters and materialize everything in the past - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING); CoordinatorActionBean.Status[] expectedStatuses = new CoordinatorActionBean.Status[19]; Arrays.fill(expectedStatuses, CoordinatorActionBean.Status.WAITING); @@ -761,7 +830,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, -1, "10", CoordinatorJob.Execution.LAST_ONLY); // We're starting from "now" this time (i.e. present/future), so it should materialize things normally - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING); expectedStatuses = new CoordinatorActionBean.Status[6]; Arrays.fill(expectedStatuses, CoordinatorActionBean.Status.WAITING); @@ -771,15 +840,15 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { public void testCurrentTimeCheck() throws Exception { long now = System.currentTimeMillis(); Date startTime = DateUtils.toDate(new Timestamp(now)); // now - Date endTime = DateUtils.toDate(new Timestamp(now + 3 * 60 * 60 * 1000)); // 3 hours from now + Date endTime = DateUtils.toDate(new Timestamp(now + 3 * 60 * 60 * 1000)); // 3 secondsFromHours from now CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, null, "5", 20); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); checkCoordJobs(job.getId(), CoordinatorJob.Status.RUNNING); job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId()); assertEquals(job.getLastActionNumber(), 12); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); // unfortunatily XCommand doesn't throw exception on precondition // assertEquals(e.getErrorCode(), ErrorCode.E1100); // assertTrue(e.getMessage().contains("Request is for future time. Lookup time is")); @@ -833,7 +902,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setEndTime(endTime); job.setMatThrottling(10); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId()); assertEquals(job.getLastActionNumber(), 3); @@ -880,7 +949,7 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { job.setMatThrottling(10); CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job); - new CoordMaterializeTransitionXCommand(job.getId(), 3600).call(); + new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call(); job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId()); assertEquals(4, job.getLastActionNumber()); @@ -1083,20 +1152,6 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { } } - private long getDSTOffset(TimeZone tz, Date d1, Date d2) { - if (tz.inDaylightTime(d1) && !tz.inDaylightTime(d2)) { - Calendar cal = Calendar.getInstance(tz); - cal.setTime(d1); - return cal.get(Calendar.DST_OFFSET); - } - if (!tz.inDaylightTime(d1) && tz.inDaylightTime(d2)) { - Calendar cal = Calendar.getInstance(tz); - cal.setTime(d2); - return cal.get(Calendar.DST_OFFSET); - } - return 0; - } - /** * Test a coordinator SLA define EL functions as variable * http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java b/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java new file mode 100644 index 0000000..1427e08 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/command/coord/TestDaylightOffsetCalculator.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.command.coord; + +import org.apache.oozie.util.DateUtils; +import org.junit.Test; + +import java.text.ParseException; +import java.util.Calendar; +import java.util.TimeZone; + +import static org.junit.Assert.assertEquals; + +public class TestDaylightOffsetCalculator { + private final static TimeZone TZ_DST = TimeZone.getTimeZone("America/Los_Angeles"); + private final static TimeZone TZ_STANDARD = TimeZone.getTimeZone("Asia/Kolkata"); + + @Test + public void testCalculateBetweenNonDSTStartAndEndGivesZeroOffset() throws ParseException { + final DaylightOffsetCalculator nonDSTStartAndEndCalculator = new DaylightOffsetCalculator( + DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"), + DateUtils.parseDateOozieTZ("2014-01-18T00:00Z")); + + final Calendar target = Calendar.getInstance(); + target.setTime(DateUtils.parseDateOozieTZ("2014-01-18T00:00Z")); + + assertEquals(nonDSTStartAndEndCalculator.calculate(TZ_DST, target), target); + } + + @Test + public void testCalculateBetweenDSTStartAndStandardEndGivesPositiveOffset() throws ParseException { + final DaylightOffsetCalculator dstStartAndStandardEndCalculator = new DaylightOffsetCalculator( + DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), + DateUtils.parseDateOozieTZ("2014-01-18T00:00Z")); + + final Calendar standardEnd = Calendar.getInstance(); + standardEnd.setTime(DateUtils.parseDateOozieTZ("2014-01-18T00:00Z")); + + final Calendar standardEndPlusOneHour = (Calendar) standardEnd.clone(); + standardEndPlusOneHour.add(Calendar.HOUR, 1); + + assertEquals(dstStartAndStandardEndCalculator.calculate(TZ_DST, standardEnd), standardEndPlusOneHour); + } + + @Test + public void testCalculateBetweenStandardStartAndDSTEndGivesNegativeOffset() throws ParseException { + final DaylightOffsetCalculator standardStartAndDSTEndCalculator = new DaylightOffsetCalculator( + DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"), + DateUtils.parseDateOozieTZ("2013-07-18T00:00Z")); + + final Calendar standardEnd = Calendar.getInstance(); + standardEnd.setTime(DateUtils.parseDateOozieTZ("2013-07-18T00:00Z")); + + final Calendar standardEndMinusOneHour = (Calendar) standardEnd.clone(); + standardEndMinusOneHour.add(Calendar.HOUR, -1); + + assertEquals(standardStartAndDSTEndCalculator.calculate(TZ_DST, standardEnd), standardEndMinusOneHour); + } + + @Test + public void testCalculateGivenStandardTZGivesZeroOffset() throws ParseException { + final DaylightOffsetCalculator standardStartAndDSTEndCalculator = new DaylightOffsetCalculator( + DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"), + DateUtils.parseDateOozieTZ("2013-07-18T00:00Z")); + + final Calendar dstStart = Calendar.getInstance(); + dstStart.setTime(DateUtils.parseDateOozieTZ("2013-01-18T00:00Z")); + + final Calendar standardEnd = Calendar.getInstance(); + standardEnd.setTime(DateUtils.parseDateOozieTZ("2013-07-18T00:00Z")); + + assertEquals(standardStartAndDSTEndCalculator.calculate(TZ_STANDARD, standardEnd), standardEnd); + } + + @Test + public void testGetDSTOffsetOfNonDaylightChangeGivesZeroOffset() throws ParseException { + final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST, + DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), + DateUtils.parseDateOozieTZ("2013-07-19T00:00Z")); + + assertEquals("non-daylight change should give zero DST offset", 0L, dstOffset); + } + + @Test + public void testGetDSTOffsetOfDaylightChangeFromDSTToStandardGivesPositiveOffset() throws ParseException { + final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST, + DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"), + DateUtils.parseDateOozieTZ("2014-01-19T00:00Z")); + + assertEquals("daylight change from DST to standard should give positive DST offset", 3_600_000L, dstOffset); + } + + @Test + public void testGetDSTOffsetOfDaylightChangeFromStandardToDSTGivesNegativeOffset() throws ParseException { + final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_DST, + DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"), + DateUtils.parseDateOozieTZ("2013-07-19T00:00Z")); + + assertEquals("daylight change from standard to DST should give negative DST offset", -3_600_000L, dstOffset); + } + + @Test + public void testGetDSTOffsetGivenStandardTZGivesZeroOffset() throws ParseException { + final long dstOffset = DaylightOffsetCalculator.getDSTOffset(TZ_STANDARD, + DateUtils.parseDateOozieTZ("2013-01-18T00:00Z"), + DateUtils.parseDateOozieTZ("2013-07-19T00:00Z")); + + assertEquals("daylight change from standard to DST should give negative DST offset", 0L, dstOffset); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/91c3d0c0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d7e3057..84880ca 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2726 Flaky test due to daylight saving changes (sasishsaley, andras.piros via gezapeti) OOZIE-2645 Deprecate Instrumentation in favor of Metrics (andras.piros via gezapeti) OOZIE-3150 Remove references to not present dependencies within NOTICE.txt (gezapeti via andras.piros) OOZIE-3201 Typo in TestCoordActionInputCheckXCommand (gongchuanjie via andras.piros)
