OOZIE-2630 amend for Oozie Coordinator EL Functions to get first day of the week/month (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0b336e50 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0b336e50 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0b336e50 Branch: refs/heads/oya Commit: 0b336e50f6b731f852c273e2735621c52950bb27 Parents: d249fbf Author: Satish Subhashrao Saley <[email protected]> Authored: Mon Feb 6 16:21:58 2017 -0800 Committer: Satish Subhashrao Saley <[email protected]> Committed: Mon Feb 6 16:21:58 2017 -0800 ---------------------------------------------------------------------- .../oozie/command/coord/CoordCommandUtils.java | 110 ++++++++++++------- .../apache/oozie/coord/CoordELFunctions.java | 13 ++- .../command/coord/TestCoordCommandUtils.java | 95 +++++++++++----- .../test/resources/coord-dataset-endOfDays.xml | 2 +- .../resources/coord-dataset-endOfMonths.xml | 6 +- .../test/resources/coord-dataset-endOfWeeks.xml | 6 +- .../test/resources/out-of-phase-coordinator.xml | 35 ++++++ .../site/twiki/CoordinatorFunctionalSpec.twiki | 85 ++++++++++---- 8 files changed, 255 insertions(+), 97 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java index 3a7a930..f7f947f 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java @@ -92,10 +92,11 @@ public class CoordCommandUtils { */ public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception { int funcType = getFuncType(function); - if (funcType == ABSOLUTE || funcType == ENDOFMONTHS || funcType == ENDOFWEEKS || funcType == ENDOFDAYS) { + if (funcType == ABSOLUTE) { return funcType; } - if (funcType == CURRENT || funcType == LATEST) { + if (funcType == CURRENT || funcType == LATEST || funcType == ENDOFMONTHS || funcType == ENDOFWEEKS + || funcType == ENDOFDAYS) { return parseOneArg(function); } else { @@ -268,13 +269,16 @@ public class CoordCommandUtils { parseOneStringArg(strStart)); } else if (funcType == ENDOFMONTHS) { - resolveInstanceRangeEndOfMonths(event, instances, appInst, conf, eval, strStart, endIndex); + resolveInstanceRangeEndOfDuration(TimeUnit.MONTH, event, instances, appInst, conf, eval, strStart, + startIndex, endIndex); } else if (funcType == ENDOFWEEKS) { - resolveInstanceRangeEndOfWeeks(event, instances, appInst, conf, eval, strStart, endIndex); + resolveInstanceRangeEndOfDuration(TimeUnit.WEEK, event, instances, appInst, conf, eval, strStart, + startIndex, endIndex); } else if (funcType == ENDOFDAYS) { - resolveInstanceRangeEndOfDays(event, instances, appInst, conf, eval, strStart, endIndex); + resolveInstanceRangeEndOfDuration(TimeUnit.DAY, event, instances, appInst, conf, eval, strStart, + startIndex, endIndex); } else { if (funcType == OFFSET) { @@ -340,40 +344,6 @@ public class CoordCommandUtils { } } - private static void resolveInstanceRangeEndOfDays(Element event, StringBuilder instances, SyncCoordAction appInst, - Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception { - Date nominalTime = appInst.getNominalTime(); - Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime)); - cal.set(Calendar.HOUR_OF_DAY, 0); - int diff = Integer.parseInt(parseOneStringArg(strStart)); - cal.add(Calendar.DATE, diff); - resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, - DateUtils.formatDateOozieTZ(cal.getTime())); - } - - private static void resolveInstanceRangeEndOfWeeks(Element event, StringBuilder instances, SyncCoordAction appInst, - Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception { - Date nominalTime = appInst.getNominalTime(); - Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime)); - cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek()); - int diff = Integer.parseInt(parseOneStringArg(strStart)); - cal.add(Calendar.WEEK_OF_YEAR, diff); - resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, - DateUtils.formatDateOozieTZ(cal.getTime())); - } - - private static void resolveInstanceRangeEndOfMonths(Element event, StringBuilder instances, SyncCoordAction appInst, - Configuration conf, ELEvaluator eval, String strStart, int endIndex) throws Exception { - int FIRST_DAY_OF_MONTH = 1; - Date nominalTime = appInst.getNominalTime(); - Calendar cal = DateUtils.getCalendar(DateUtils.formatDateOozieTZ(nominalTime)); - cal.set(Calendar.DAY_OF_MONTH, FIRST_DAY_OF_MONTH); - int diff = Integer.parseInt(parseOneStringArg(strStart)); - cal.add(Calendar.MONTH, diff); - resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, - DateUtils.formatDateOozieTZ(cal.getTime())); - } - private static void resolveAbsoluteRange(Element event, StringBuilder instances, SyncCoordAction appInst, Configuration conf, ELEvaluator eval, String strStart, int endIndex, String rangeStr) throws Exception { StringBuffer bf = new StringBuffer(); @@ -388,6 +358,15 @@ public class CoordCommandUtils { } } + private static void resolveInstanceRangeEndOfDuration(TimeUnit duration, Element event, StringBuilder instances, + SyncCoordAction appInst, Configuration conf, ELEvaluator eval, String strStart, int startIndex, + int endIndex) throws Exception { + Calendar startInstance = new StartInstanceFinder(startIndex, duration, CoordELFunctions.getDatasetTZ(eval), + appInst.getNominalTime()).getStartInstance(); + resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, + DateUtils.formatDateOozieTZ(startInstance)); + } + /** * Materialize one instance like current(-2) * @@ -954,4 +933,57 @@ public class CoordCommandUtils { } return firstMissingDependencies; } + + /** + * Class to find get start instance + */ + private static class StartInstanceFinder { + + private int startIndex; + private TimeUnit timeUnit; + private TimeZone datasetTimeZone; + private Date nominalTime; + + /** + * @param startIndex dataset index + * @param timeUnit + * @param datasetTimeZone + * @param nominalTime nominal time of action + */ + public StartInstanceFinder(int startIndex, TimeUnit timeUnit, TimeZone datasetTimeZone, Date nominalTime) { + this.startIndex = startIndex; + this.timeUnit = timeUnit; + this.datasetTimeZone = datasetTimeZone; + this.nominalTime = nominalTime; + } + + /** + * Calculates the start instance. It put the start instance to the start + * of a day i.e. 00:00:00. + */ + public Calendar getStartInstance() throws Exception { + Calendar startInstance = Calendar.getInstance(datasetTimeZone); + startInstance.setTime(nominalTime); + startInstance.set(Calendar.HOUR_OF_DAY, 0); + startInstance.set(Calendar.MINUTE, 0); + startInstance.set(Calendar.SECOND, 0); + switch (timeUnit) { + case WEEK: + startInstance.set(Calendar.DAY_OF_WEEK, startInstance.getFirstDayOfWeek()); + startInstance.add(Calendar.WEEK_OF_YEAR, startIndex + 1); + break; + case MONTH: + int FIRST_DAY_OF_MONTH = 1; + startInstance.set(Calendar.DAY_OF_MONTH, FIRST_DAY_OF_MONTH); + startInstance.add(Calendar.MONTH, startIndex + 1); + break; + case DAY: + startInstance.add(Calendar.DATE, startIndex + 1); + break; + default: + } + return startInstance; + } + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java index bc18f4d..5b4dae4 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java @@ -848,14 +848,21 @@ public class CoordELFunctions { public static String ph2_coord_absolute_range(String startInstance, int end) throws Exception { int[] instanceCount = new int[1]; - + Calendar startInstanceCal = DateUtils.getCalendar(startInstance); + Calendar currentInstance = getCurrentInstance(startInstanceCal.getTime(), instanceCount); // getCurrentInstance() returns null, which means startInstance is less // than initial instance - if (getCurrentInstance(DateUtils.getCalendar(startInstance).getTime(), instanceCount) == null) { + if (currentInstance == null) { throw new CommandException(ErrorCode.E1010, "intial-instance should be equal or earlier than the start-instance. intial-instance is " + getInitialInstance() + " and start-instance is " + startInstance); } + if (currentInstance.getTimeInMillis() != startInstanceCal.getTimeInMillis()) { + throw new CommandException(ErrorCode.E1010, + "initial-instance is not in phase with start-instance. initial-instance is " + + DateUtils.formatDateOozieTZ(getInitialInstanceCal()) + " and start-instance is " + + DateUtils.formatDateOozieTZ(startInstanceCal)); + } int[] nominalCount = new int[1]; if (getCurrentInstance(getActionCreationtime(), nominalCount) == null) { throw new CommandException(ErrorCode.E1010, @@ -1584,7 +1591,7 @@ public class CoordELFunctions { /** * @return dataset TimeZone */ - private static TimeZone getDatasetTZ(ELEvaluator eval) { + public static TimeZone getDatasetTZ(ELEvaluator eval) { SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); if (ds == null) { throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java index 22d1f61..6df4e41 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java @@ -18,7 +18,9 @@ package org.apache.oozie.command.coord; +import java.io.File; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.text.ParseException; import java.util.Calendar; @@ -38,6 +40,7 @@ import org.apache.oozie.service.Services; import org.apache.oozie.service.UUIDService; import org.apache.oozie.test.XDataTestCase; import org.apache.oozie.util.DateUtils; +import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; import org.jdom.Element; @@ -519,30 +522,30 @@ public class TestCoordCommandUtils extends XDataTestCase { @Test public void testCoordEndOfMonthsParamerized() throws Exception { - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(0)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-1)", "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); Element e1 = (Element) elementList.get(0); Element e2 = (Element) elementList.get(1); - // startInstance = coord:endOfMonths(0) i.e.2009-08-01T01:00Z and + // startInstance = coord:endOfMonths(-1) i.e.2009-08-01T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z Calendar start = DateUtils.getCalendar("2009-08-01T01:00Z", DateUtils.UTC); checkUris(e1.getChild("uris", e1.getNamespace()).getTextTrim(), (Calendar) start.clone(), Calendar.DATE, 1, "YYYY/MM/dd"); // Test parameterized - // startInstance = coord:endOfMonths(0) i.e.2009-08-01T01:00Z and + // startInstance = coord:endOfMonths(-1) i.e.2009-08-01T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd"); } public void testCoordEndOfMonthsStartingFromPrevMonth() throws Exception { - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-1)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-2)", "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); Element e2 = (Element) elementList.get(1); e2 = (Element) elementList.get(1); - // startInstance = coord:endOfMonths(0) i.e.2009-07-01T01:00Z and + // startInstance = coord:endOfMonths(-2) i.e.2009-07-01T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z Calendar start = DateUtils.getCalendar("2009-07-01T01:00Z", DateUtils.UTC); checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd"); @@ -552,7 +555,7 @@ public class TestCoordCommandUtils extends XDataTestCase { try { // start instance = 2009-10-01T01:00Z // end instance = 2009-08-01T01:00Z - getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(2)", + getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(1)", "coord:current(0)", "2009-08-06T01:00Z", "2009-08-16T01:00Z"); fail("Should throw exception because end is earlier than start-instance"); } @@ -566,7 +569,7 @@ public class TestCoordCommandUtils extends XDataTestCase { try { // start instance = 2009-06-01T01:00Z // initial instance = 2009-08-06T01:00Z - getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-2)", "coord:current(0)", + getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-3)", "coord:current(0)", "2009-08-06T01:00Z", "2009-08-16T01:00Z"); fail("Should throw exception because initial instance is later than start instance"); } @@ -579,7 +582,7 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfMonthsFailOnLatestAsEndInstance() throws Exception { try { - getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(0)", + getSyncDatasetEvents("coord-dataset-endOfMonths.xml", "coord:endOfMonths(-1)", "coord:latest(2)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); fail("Should throw exception because latest is not allowed as end-instance"); } @@ -590,16 +593,28 @@ public class TestCoordCommandUtils extends XDataTestCase { } } + public void testCoordEndOfMonthsInitialInstaceNotInPhase() throws Exception { + try { + getSyncDatasetEvents("out-of-phase-coordinator.xml", "${coord:endOfMonths(-1)}", "${coord:current(0)}", + "2009-08-20T18:00Z", "2009-08-20T18:00Z"); + fail("Should throw exception because initial instance is not in phase with start instance"); + } + catch (Exception e) { + e.printStackTrace(System.out); + assertTrue(e.getCause().getMessage().contains("initial-instance is not in phase with start-instance")); + } + } + @Test public void testCoordEndOfWeeksParamerized() throws Exception { String nominalTime = "2009-08-20T01:00Z"; String startTime = nominalTime; - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(0)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-1)", "coord:current(0)", nominalTime, startTime); Element e1 = (Element) elementList.get(0); Element e2 = (Element) elementList.get(1); - // startInstance = coord:endOfWeeks(0) + // startInstance = coord:endOfWeeks(-1) // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z Calendar start = DateUtils.getCalendar(nominalTime, DateUtils.UTC); start.add(Calendar.DAY_OF_WEEK, start.getFirstDayOfWeek() - start.get(Calendar.DAY_OF_WEEK)); @@ -607,18 +622,18 @@ public class TestCoordCommandUtils extends XDataTestCase { "YYYY/MM/dd"); // Test parameterized - // startInstance = coord:endOfWeeks(0) + // startInstance = coord:endOfWeeks(-1) // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.DATE, 1, "YYYY/MM/dd"); } public void testCoordEndOfWeeksStartingFromPrevWeek() throws Exception { - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-1)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-2)", "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); Element e2 = (Element) elementList.get(1); e2 = (Element) elementList.get(1); - // startInstance = coord:endOfWeeks(-1) + // startInstance = coord:endOfWeeks(-2) // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z Calendar start = DateUtils.getCalendar("2009-08-10T01:00Z", DateUtils.UTC); start.add(Calendar.DAY_OF_WEEK, start.getFirstDayOfWeek() - start.get(Calendar.DAY_OF_WEEK)); @@ -627,9 +642,9 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfWeeksFailOnStartInstanceIsLater() throws Exception { try { - // start instance = coord:endOfWeeks(2) i.e. 2009-08-30T01:00Z + // start instance = coord:endOfWeeks(1) i.e. 2009-08-30T01:00Z // end instance = coord:current(0) i.e. 2009-08-20T01:00Z - getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(2)", "coord:current(0)", + getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(1)", "coord:current(0)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); fail("Should throw exception because end is earlier than start-instance"); } @@ -641,9 +656,9 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfWeeksFailOnInitialInstanceIsLater() throws Exception { try { - // start instance = coord:endOfWeeks(-10) i.e. 2009-05-31T01:00Z + // start instance = coord:endOfWeeks(-11) i.e. 2009-05-31T01:00Z // initial instance = 2009-06-06T01:00Z - getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-10)", "coord:current(0)", + getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-11)", "coord:current(0)", "2009-08-06T01:00Z", "2009-08-16T01:00Z"); fail("Should throw exception because initial instance is later than start instance"); } @@ -656,7 +671,7 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfWeeksFailOnLatestAsEndInstance() throws Exception { try { - getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(0)", "coord:latest(2)", + getSyncDatasetEvents("coord-dataset-endOfWeeks.xml", "coord:endOfWeeks(-1)", "coord:latest(2)", "2009-08-20T01:00Z", "2009-08-20T01:00Z"); fail("Should throw exception because latest is not allowed as end-instance"); } @@ -667,32 +682,44 @@ public class TestCoordCommandUtils extends XDataTestCase { } } + public void testCoordEndOfWeeksInitialInstaceNotInPhase() throws Exception { + try { + getSyncDatasetEvents("out-of-phase-coordinator.xml", "${coord:endOfWeeks(-1)}", "${coord:current(0)}", + "2009-08-20T18:00Z", "2009-08-20T18:00Z"); + fail("Should throw exception because initial instance is not in phase with start instance"); + } + catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("initial-instance is not in phase with start-instance")); + } + } + @Test public void testCoordEndOfDaysParameterized() throws Exception { - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(0)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", + "coord:endOfDays(-1)", "coord:current(0)", "2009-08-20T18:00Z", "2009-08-20T18:00Z"); Element e1 = (Element) elementList.get(0); Element e2 = (Element) elementList.get(1); - // startInstance = coord:endOfDays(0) i.e 2009-08-20T01:00Z and + // startInstance = coord:endOfDays(-1) i.e 2009-08-20T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T18:00Z Calendar start = DateUtils.getCalendar("2009-08-20T00:00Z", DateUtils.UTC); checkUris(e1.getChild("uris", e1.getNamespace()).getTextTrim(), (Calendar) start.clone(), Calendar.MINUTE, 30, "YYYY/MM/dd/HH/mm"); // Test parameterized - // startInstance = coord:endOfDays(0) i.e 2009-08-20T01:00Z and + // startInstance = coord:endOfDays(-1) i.e 2009-08-20T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.MINUTE, 30, "YYYY/MM/dd/HH/mm"); } public void testCoordEndOfDaysStartingFromPrevDay() throws Exception { - List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-1)", + List<?> elementList = getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-2)", "coord:current(0)", "2009-08-20T18:00Z", "2009-08-20T18:00Z"); Element e2 = (Element) elementList.get(1); - // startInstance = coord:endOfDays(-1) i.e 2009-08-19T01:00Z and + // startInstance = coord:endOfDays(-2) i.e 2009-08-19T01:00Z and // endInstance = coord:current(0) i.e. 2009-08-20T01:00Z Calendar start = DateUtils.getCalendar("2009-08-19T00:00Z", DateUtils.UTC); checkUris(e2.getChild("uris", e2.getNamespace()).getTextTrim(), start, Calendar.MINUTE, 30, @@ -701,9 +728,9 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfDaysFailOnStartInstanceIsLater() throws Exception { try { - // start instance = coord:endOfDays(2) i.e. 2009-08-08T00:00Z + // start instance = coord:endOfDays(1) i.e. 2009-08-08T00:00Z // end instance = coord:current(0) i.e. 2009-08-06T18:00Z - getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(2)", + getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(1)", "coord:current(0)", "2009-08-06T18:00Z", "2009-08-16T18:00Z"); } catch (Exception e) { @@ -714,9 +741,9 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfDaysFailOnInitialInstanceIsLater() throws Exception { try { - // start instance = coord:endOfDays(-10) i.e. 2009-07-27T00:00Z + // start instance = coord:endOfDays(-11) i.e. 2009-07-27T00:00Z // initial instance = 2009-08-01T01:00Z - getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-10)", "coord:current(0)", + getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-11)", "coord:current(0)", "2009-08-06T18:00Z", "2009-08-16T18:00Z"); fail("Should throw exception because initial instance is later than start instance"); } @@ -728,7 +755,7 @@ public class TestCoordCommandUtils extends XDataTestCase { public void testCoordEndOfDaysFailOnLatestAsEndInstance() throws Exception { try { - getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(0)", + getSyncDatasetEvents("coord-dataset-endOfDays.xml", "coord:endOfDays(-1)", "coord:latest(2)", "2009-08-20T18:00Z", "2009-08-20T18:00Z"); fail("Should throw exception. Start-instance is coord:endOfDays and end-instance is latest"); } @@ -740,6 +767,17 @@ public class TestCoordCommandUtils extends XDataTestCase { } } + public void testCoordEndOfDaysInitialInstaceNotInPhase() throws Exception { + try { + getSyncDatasetEvents("out-of-phase-coordinator.xml", "${coord:endOfDays(-1)}", "${coord:current(0)}" + , "2009-08-20T18:00Z", "2009-08-20T18:00Z"); + fail("Should throw exception because initial instance is not in phase with start instance"); + } + catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("initial-instance is not in phase with start-instance")); + } + } + protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, String freq) throws Exception { CoordinatorJobBean coordJob = createCoordJob(status, startTime, endTime, false, false, 0); @@ -854,4 +892,5 @@ public class TestCoordCommandUtils extends XDataTestCase { .getChildren(); return elementList; } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/test/resources/coord-dataset-endOfDays.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-dataset-endOfDays.xml b/core/src/test/resources/coord-dataset-endOfDays.xml index 6aa3f00..f59c277 100644 --- a/core/src/test/resources/coord-dataset-endOfDays.xml +++ b/core/src/test/resources/coord-dataset-endOfDays.xml @@ -24,7 +24,7 @@ <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE};region=us </uri-template> </dataset> - <start-instance>${coord:endOfDays(0)} + <start-instance>${coord:endOfDays(-1)} </start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/test/resources/coord-dataset-endOfMonths.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-dataset-endOfMonths.xml b/core/src/test/resources/coord-dataset-endOfMonths.xml index 0ea5cac..d380844 100644 --- a/core/src/test/resources/coord-dataset-endOfMonths.xml +++ b/core/src/test/resources/coord-dataset-endOfMonths.xml @@ -19,17 +19,17 @@ </controls> <input-events> <data-in name="input" dataset="logs"> - <dataset name='logs' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='logs' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us </uri-template> </dataset> - <start-instance>${coord:endOfMonths(0)} + <start-instance>${coord:endOfMonths(-1)} </start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> <data-in name="input" dataset="test"> - <dataset name='test' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='test' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us </uri-template> http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/test/resources/coord-dataset-endOfWeeks.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-dataset-endOfWeeks.xml b/core/src/test/resources/coord-dataset-endOfWeeks.xml index 7879bf4..870708e 100644 --- a/core/src/test/resources/coord-dataset-endOfWeeks.xml +++ b/core/src/test/resources/coord-dataset-endOfWeeks.xml @@ -19,17 +19,17 @@ </controls> <input-events> <data-in name="input" dataset="logs"> - <dataset name='logs' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='logs' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us </uri-template> </dataset> - <start-instance>${coord:endOfWeeks(0)} + <start-instance>${coord:endOfWeeks(-1)} </start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> <data-in name="input" dataset="test"> - <dataset name='test' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='test' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us </uri-template> http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/core/src/test/resources/out-of-phase-coordinator.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/out-of-phase-coordinator.xml b/core/src/test/resources/out-of-phase-coordinator.xml new file mode 100644 index 0000000..6758d81 --- /dev/null +++ b/core/src/test/resources/out-of-phase-coordinator.xml @@ -0,0 +1,35 @@ +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<coordinator-app xmlns="uri:oozie:coordinator:0.3" name="test-coord" +frequency="1" freq_timeunit="DAY" start="2009-08-20T01:00Z" end="2009-10-10T23:59Z" +timezone="UTC" end_of_duration="NONE"> + <controls> + <timeout>10</timeout> + <concurrency>2</concurrency> + <execution>LIFO</execution> + </controls> + <input-events> + <data-in name="input" dataset="logs"> + <dataset name="logs" frequency="30" initial-instance="2008-08-20T01:17Z" + timezone="UTC" freq_timeunit="MINUTE" end_of_duration="NONE"> + <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE};region=us</uri-template> + </dataset> + <start-instance>${startInstance}</start-instance> + <end-instance>${endInstance}</end-instance> + </data-in> + </input-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows/</app-path> + </workflow> + </action> +</coordinator-app> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/0b336e50/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 4b21ad9..3828eb5 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -2271,27 +2271,50 @@ The dataset instances for the input events for the coordinator action at second =${coord:endOfMonths(int n)}= represents dataset instance at start of n <sup>th</sup> month. coord:endOfMonths is only supported with range, where start-instance is coord:endOfMonths and end-instance is coord:current. Specifying start of a month is useful if you want to process all the dataset instances from starting of a month to the current instance. -This function calculates the dataset occurences similar to -[[CoordinatorFunctionalSpec#a4.4.2.2._The_coord:endOfMonthsint_n_EL_function][coord:endOfMonths(int n) EL function]] -mentioned above. +*%GREEN% Examples: %ENDCOLOR%* + +1. *. =${coord:endOfMonths(int n)}= datetime calculation:* + +Datasets Definition: + +<verbatim> +<datasets> +. + <dataset name="logs" frequency="${coord:days(1)}" + initial-instance="2009-01-01T00:00Z" timezone="UTC"> + <uri-template>hdfs://bar:8020/app/logs/${YEAR}${MONTH}/${DAY}</uri-template> + </dataset> +. +</datasets> +</verbatim> + +For a coordinator action creation time: =2009-05-29T00:00Z= the =${coord:endOfMonths(int n)}= EL function +would resolve to the following datetime values for the 'logs' dataset: + +| *${coord:endOfMonths(int offset)}* | *Dataset 'logs'* | *Comments* | +| =${coord:endOfMonths(-1)}= | =2009-05-01T00:00Z= | End of last month i.e. start of current month | +| =${coord:endOfMonths(0)}= | =2009-06-01T00:00Z= | End of current month i.e. start of next month | +| =${coord:endOfMonths(-2)}= | =2009-04-01T00:00Z= | | +| =${coord:endOfMonths(-3)}= | =2009-03-01T00:00Z= | | + *%GREEN% Example: %ENDCOLOR%*: Coordinator application definition: <verbatim> <coordinator-app name="app-coord" frequency="${coord:days(1)}" - start='2009-08-06T01:00Z' end="2009-12-31T00:00" timezone="UTC" + start='2009-08-06T00:00Z' end="2009-12-31T00:00" timezone="UTC" xmlns="uri:oozie:coordinator:0.4"> <input-events> <data-in name="input" dataset="logs"> - <dataset name='a' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='a' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY} </uri-template> </dataset> - <start-instance>${coord:endOfMonths(0)}</start-instance> + <start-instance>${coord:endOfMonths(-1)}</start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> </input-events> @@ -2356,14 +2379,14 @@ Datasets Definition: </datasets> </verbatim> -For a coordinator action creation time: =2009-05-29T00:00Z= the =${coord:endOfMonths(int n)}= EL function +For a coordinator action creation time: =2009-05-29T00:00Z= the =${coord:endOfWeeks(int n)}= EL function would resolve to the following datetime values for the 'logs' dataset: -| *${coord:endOfWeeks(int offset)}* | *Dataset 'logs'* | -| =${coord:endOfWeeks(0)}= | =2009-05-24T00:00Z= | -| =${coord:endOfWeeks(1)}= | =2009-05-31T00:00Z= | -| =${coord:endOfWeeks(-1)}= | =2009-05-17T00:00Z= | -| =${coord:endOfWeeks(-3)}= | =2009-05-03T00:00Z= | +| *${coord:endOfWeeks(int offset)}* | *Dataset 'logs'* | *Comments* | +| =${coord:endOfWeeks(-1)}= | =2009-05-24T00:00Z= | End of last week i.e. start of current week | +| =${coord:endOfWeeks(0)}= | =2009-05-31T00:00Z= | End of current week i.e. start of next week | +| =${coord:endOfWeeks(-2)}= | =2009-05-17T00:00Z= | | +| =${coord:endOfWeeks(-4)}= | =2009-05-03T00:00Z= | | *%GREEN% Example: %ENDCOLOR%*: @@ -2372,16 +2395,16 @@ Coordinator application definition: <verbatim> <coordinator-app name="app-coord" frequency="${coord:days(1)}" - start='2009-08-06T01:00Z' end="2009-12-31T00:00" timezone="UTC" + start='2009-08-06T00:00Z' end="2009-12-31T00:00" timezone="UTC" xmlns="uri:oozie:coordinator:0.4"> <input-events> <data-in name="input" dataset="logs"> - <dataset name='a' frequency='1' initial-instance='2009-06-06T01:00Z' + <dataset name='a' frequency='1' initial-instance='2009-06-06T00:00Z' timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY} </uri-template> </dataset> - <start-instance>${coord:endOfWeeks(0)}</start-instance> + <start-instance>${coord:endOfWeeks(-1)}</start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> </input-events> @@ -2421,10 +2444,32 @@ The dataset instances for the input events for the coordinator action at second =${coord:endOfDays(int n)}= represents dataset instance at start of n <sup>th</sup> day. coord:endOfDays is only supported with range, where start-instance is coord:endOfDays and end-instance is coord:current. Specifying start of a day is useful if you want to process all the dataset instances from starting of a day to the current instance. -This function calculates the dataset occurences similar to -[[CoordinatorFunctionalSpec#a4.4.1.2._The_coord:endOfDaysint_n_EL_function][coord:endOfDays(int n) EL function]] -mentioned above. +*%GREEN% Examples: %ENDCOLOR%* + +1. *. =${coord:endOfDays(int n)}= datetime calculation:* + +Datasets Definition: + +<verbatim> +<datasets> +. + <dataset name="logs" frequency="${coord:days(1)}" + initial-instance="2009-01-01T00:00Z" timezone="UTC"> + <uri-template>hdfs://bar:8020/app/logs/${YEAR}${MONTH}/${DAY}/${HOUR}/${MINUTE}</uri-template> + </dataset> +. +</datasets> +</verbatim> + +For a coordinator action creation time: =2009-05-25T23:00Z= the =${coord:endOfDays(int n)}= EL function +would resolve to the following datetime values for the 'logs' dataset: + +| *${coord:endOfDays(int offset)}* | *Dataset 'logs'* | *Comments* | +| =${coord:endOfDays(-1)}= | =2009-05-25T00:00Z= | End of previous day i.e. start of the current day | +| =${coord:endOfDays(0)}= | =2009-05-26T00:00Z= | End of current day i.e. start of the next day | +| =${coord:endOfDays(-2)}= | =2009-05-24T00:00Z= | | +| =${coord:endOfDays(-4)}= | =2009-05-22T00:00Z= | | *%GREEN% Example: %ENDCOLOR%*: @@ -2432,7 +2477,7 @@ Coordinator application definition: <verbatim> <coordinator-app name="app-coord" frequency='60' freq_timeunit='MINUTE' - start='2009-08-06T05:00Z' end="2009-12-31T00:00" timezone="UTC" + start='2009-08-06T00:00Z' end="2009-12-31T00:00" timezone="UTC" xmlns="uri:oozie:coordinator:0.4"> <input-events> <data-in name="input" dataset="logs"> @@ -2441,7 +2486,7 @@ Coordinator application definition: <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE} </uri-template> </dataset> - <start-instance>${coord:endOfDays(0)}</start-instance> + <start-instance>${coord:endOfDays(-1)}</start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> </input-events>
