Updated Branches: refs/heads/master 0862e8657 -> 3a0a13a0c
OOZIE-1504 Allow specifying a fixed instance as the start instance of a data-in (puru via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3a0a13a0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3a0a13a0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3a0a13a0 Branch: refs/heads/master Commit: 3a0a13a0cbb8cb42d3a906c5dfc1ede5877e032f Parents: 0862e86 Author: Rohini Palaniswamy <[email protected]> Authored: Sun Dec 29 16:44:35 2013 -0800 Committer: Rohini Palaniswamy <[email protected]> Committed: Sun Dec 29 16:44:35 2013 -0800 ---------------------------------------------------------------------- .../oozie/command/coord/CoordCommandUtils.java | 135 ++++++++++++------- .../apache/oozie/coord/CoordELFunctions.java | 46 ++++++- core/src/main/resources/oozie-default.xml | 11 +- .../command/coord/TestCoordCommandUtils.java | 123 ++++++++++++++++- .../oozie/coord/TestCoordELFunctions.java | 32 ++++- .../test/resources/coord-dataset-absolute.xml | 48 +++++++ .../site/twiki/CoordinatorFunctionalSpec.twiki | 55 +++++++- docs/src/site/twiki/WebServicesAPI.twiki | 1 + release-log.txt | 1 + 9 files changed, 390 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java index 7495395..b00eb7b 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java @@ -59,6 +59,7 @@ public class CoordCommandUtils { public static int LATEST = 1; public static int FUTURE = 2; public static int OFFSET = 3; + public static int ABSOLUTE = 4; public static int UNEXPECTED = -1; public static final String RESOLVED_UNRESOLVED_SEPARATOR = "!!"; public static final String UNRESOLVED_INST_TAG = "unresolved-instances"; @@ -74,6 +75,9 @@ public class CoordCommandUtils { */ public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception { int funcType = getFuncType(function); + if (funcType == ABSOLUTE) { + return ABSOLUTE; + } if (funcType == CURRENT || funcType == LATEST) { return parseOneArg(function); } @@ -109,6 +113,15 @@ public class CoordCommandUtils { throw new RuntimeException("Unformatted function :" + funcName); } + public static String parseOneStringArg(String funcName) throws Exception { + int firstPos = funcName.indexOf("("); + int lastPos = funcName.lastIndexOf(")"); + if (firstPos >= 0 && lastPos > firstPos) { + return funcName.substring(firstPos + 1, lastPos).trim(); + } + throw new RuntimeException("Unformatted function :" + funcName); + } + private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception { int firstPos = funcName.indexOf("("); int secondPos = funcName.lastIndexOf(","); @@ -140,6 +153,9 @@ public class CoordCommandUtils { else if (function.indexOf("offset") >= 0) { return OFFSET; } + else if (function.indexOf("absolute") >= 0) { + return ABSOLUTE; + } return UNEXPECTED; // throw new RuntimeException("Unexpected instance name "+ function); } @@ -151,12 +167,22 @@ public class CoordCommandUtils { */ public static void checkIfBothSameType(String startInst, String endInst) throws CommandException { if (getFuncType(startInst) != getFuncType(endInst)) { - throw new CommandException(ErrorCode.E1010, - " start-instance and end-instance both should be either latest or current or future or offset\n" - + " start " + startInst + " and end " + endInst); + if (getFuncType(startInst) == ABSOLUTE) { + if (getFuncType(endInst) != CURRENT) { + throw new CommandException(ErrorCode.E1010, + "Only start-instance as absolute and end-instance as current is supported." + " start = " + + startInst + " end = " + endInst); + } + } + else { + throw new CommandException(ErrorCode.E1010, + " start-instance and end-instance both should be either latest or current or future or offset\n" + + " start " + startInst + " and end " + endInst); + } } } + /** * Resolve list of <instance> </instance> tags. * @@ -170,6 +196,7 @@ public class CoordCommandUtils { public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst, Configuration conf, ELEvaluator eval) throws Exception { for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) { + if (instances.length() > 0) { instances.append(CoordELFunctions.INSTANCE_SEPARATOR); } @@ -207,57 +234,73 @@ public class CoordCommandUtils { int endIndex = getInstanceNumber(strEnd, restArg); String endRestArg = restArg.toString(); int funcType = getFuncType(strStart); - if (funcType == OFFSET) { - TimeUnit startU = TimeUnit.valueOf(startRestArg); - TimeUnit endU = TimeUnit.valueOf(endRestArg); - if (startU.getCalendarUnit() * startIndex > endU.getCalendarUnit() * endIndex) { - throw new CommandException(ErrorCode.E1010, - " start-instance should be equal or earlier than the end-instance \n" - + XmlUtils.prettyPrint(event)); - } - Calendar startCal = CoordELFunctions.resolveOffsetRawTime(startIndex, startU, eval); - Calendar endCal = CoordELFunctions.resolveOffsetRawTime(endIndex, endU, eval); - if (startCal != null && endCal != null) { - List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval); - for (int i = expandedFreqs.size() - 1; i >= 0; i--) { - String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i) - + ", \"MINUTE\")}", appInst, conf, eval); - if (matInstance == null || matInstance.length() == 0) { - // Earlier than dataset's initial instance - break; - } - if (instances.length() > 0) { - instances.append(CoordELFunctions.INSTANCE_SEPARATOR); - } - instances.append(matInstance); + + if (funcType == ABSOLUTE) { + StringBuffer bf = new StringBuffer(); + bf.append("${coord:absoluteRange(\"").append(parseOneStringArg(strStart)) + .append("\",").append(endIndex).append(")}"); + String matInstance = materializeInstance(event, bf.toString(), appInst, conf, eval); + if (matInstance != null && !matInstance.isEmpty()) { + if (instances.length() > 0) { + instances.append(CoordELFunctions.INSTANCE_SEPARATOR); } + instances.append(matInstance); } } else { - if (startIndex > endIndex) { - throw new CommandException(ErrorCode.E1010, - " start-instance should be equal or earlier than the end-instance \n" - + XmlUtils.prettyPrint(event)); - } - if (funcType == CURRENT) { - // Everything could be resolved NOW. no latest() ELs - String matInstance = materializeInstance(event, "${coord:currentRange(" + startIndex + "," - + endIndex + ")}", appInst, conf, eval); - if (matInstance != null && !matInstance.isEmpty()) { - if (instances.length() > 0) { - instances.append(CoordELFunctions.INSTANCE_SEPARATOR); + if (funcType == OFFSET) { + TimeUnit startU = TimeUnit.valueOf(startRestArg); + TimeUnit endU = TimeUnit.valueOf(endRestArg); + if (startU.getCalendarUnit() * startIndex > endU.getCalendarUnit() * endIndex) { + throw new CommandException(ErrorCode.E1010, + " start-instance should be equal or earlier than the end-instance \n" + + XmlUtils.prettyPrint(event)); + } + Calendar startCal = CoordELFunctions.resolveOffsetRawTime(startIndex, startU, eval); + Calendar endCal = CoordELFunctions.resolveOffsetRawTime(endIndex, endU, eval); + if (startCal != null && endCal != null) { + List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval); + for (int i = expandedFreqs.size() - 1; i >= 0; i--) { + String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i) + + ", \"MINUTE\")}", appInst, conf, eval); + if (matInstance == null || matInstance.length() == 0) { + // Earlier than dataset's initial instance + break; + } + if (instances.length() > 0) { + instances.append(CoordELFunctions.INSTANCE_SEPARATOR); + } + instances.append(matInstance); } - instances.append(matInstance); } } - else { // latest(n)/future() EL is present - if (funcType == LATEST) { - instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex) - .append(")}"); + else { + if (startIndex > endIndex) { + throw new CommandException(ErrorCode.E1010, + " start-instance should be equal or earlier than the end-instance \n" + + XmlUtils.prettyPrint(event)); + } + if (funcType == CURRENT) { + // Everything could be resolved NOW. no latest() ELs + String matInstance = materializeInstance(event, "${coord:currentRange(" + startIndex + "," + + endIndex + ")}", appInst, conf, eval); + if (matInstance != null && !matInstance.isEmpty()) { + if (instances.length() > 0) { + instances.append(CoordELFunctions.INSTANCE_SEPARATOR); + } + instances.append(matInstance); + } } - else if (funcType == FUTURE) { - instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex) - .append(",'").append(endRestArg).append("')}"); + + else { // latest(n)/future() EL is present + if (funcType == LATEST) { + instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex) + .append(")}"); + } + else if (funcType == FUTURE) { + instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex) + .append(",'").append(endRestArg).append("')}"); + } } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java index bbf6a49..d73bc7d 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java @@ -25,7 +25,9 @@ import java.util.List; import java.util.TimeZone; import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.ErrorCode; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.command.CommandException; import org.apache.oozie.dependency.URIHandler.Context; import org.apache.oozie.dependency.URIHandler; import org.apache.oozie.util.DateUtils; @@ -388,13 +390,13 @@ public class CoordELFunctions { * @throws Exception if unable to format the Date object to String */ public static String ph2_coord_formatTime(String dateTimeStr, String format) - throws Exception { + throws Exception { Date dateTime = DateUtils.parseDateOozieTZ(dateTimeStr); return DateUtils.formatDateCustom(dateTime, format); } public static String ph3_coord_formatTime(String dateTimeStr, String format) - throws Exception { + throws Exception { return ph2_coord_formatTime(dateTimeStr, format); } @@ -526,7 +528,6 @@ public class CoordELFunctions { throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); } } - /** * Determine the date-time in Oozie processing timezone of the given offset from the dataset effective nominal time. <p/> It * depends on: <p> 1. Data set frequency <p/> 2. Data set Time Unit <p/> 3. Data set Time zone/DST @@ -698,6 +699,10 @@ public class CoordELFunctions { return echoUnResolved("current", n); } + public static String ph1_coord_absolute_echo(String date) { + return echoUnResolved("absolute", date); + } + public static String ph1_coord_currentRange_echo(String start, String end) { return echoUnResolved("currentRange", start + ", " + end); } @@ -718,6 +723,39 @@ public class CoordELFunctions { return echoUnResolved("offset", n + " , " + timeUnit); } + public static String ph2_coord_absolute_echo(String date) { + return echoUnResolved("absolute", date); + } + + public static String ph2_coord_absolute_range(String startInstance, int end) throws Exception { + int[] instanceCount = new int[1]; + + // getCurrentInstance() returns null, which means startInstance is less + // than initial instance + if (getCurrentInstance(DateUtils.getCalendar(startInstance).getTime(), instanceCount) == null) { + throw new CommandException(ErrorCode.E1010, + "intial-instance should be equal or earlier than the start-instance. intial-instance is " + + getInitialInstance() + " and start-instance is " + startInstance); + } + int[] nominalCount = new int[1]; + if (getCurrentInstance(getActionCreationtime(), nominalCount) == null) { + throw new CommandException(ErrorCode.E1010, + "intial-instance should be equal or earlier than the nominal time. intial-instance is " + + getInitialInstance() + " and nominal time is " + getActionCreationtime()); + } + // getCurrentInstance return offset relative to initial instance. + // start instance offset - nominal offset = start offset relative to + // nominal time-stamp. + int start = instanceCount[0] - nominalCount[0]; + if (start > end) { + throw new CommandException(ErrorCode.E1010, + "start-instance should be equal or earlier than the end-instance. startInstance is " + + startInstance + " which is equivalent to current (" + instanceCount[0] + + ") but end is specified as current (" + end + ")"); + } + return ph2_coord_currentRange(start, end); + } + public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) { return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit); } @@ -1041,7 +1079,7 @@ public class CoordELFunctions { String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag); if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) { XLog.getLog(CoordELFunctions.class) - .debug("Found latest(" + available + "): " + uriWithDoneFlag); + .debug("Found latest(" + available + "): " + uriWithDoneFlag); if (available == startOffset) { LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag); resolved = true; http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 6788d43..0eed43d 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -787,7 +787,8 @@ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_futureRange_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, - coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -947,7 +948,9 @@ coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, - coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, + coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1001,7 +1004,9 @@ coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, - coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, + coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java index b9caf41..36c2268 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java @@ -22,8 +22,6 @@ import java.io.StringReader; import java.text.ParseException; import java.util.Date; import java.util.List; -import java.util.TimeZone; - import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.client.CoordinatorJob; @@ -253,7 +251,8 @@ public class TestCoordCommandUtils extends XDataTestCase { Date startTime = DateUtils.parseDateOozieTZ("2013-07-18T00:00Z"); Date endTime = DateUtils.parseDateOozieTZ("2013-07-18T01:00Z"); - CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, "10,20 * * * *"); + CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, + "10,20 * * * *"); Date actionTime = DateUtils.parseDateOozieTZ("2013-07-18T00:15Z"); Date expectedDate = DateUtils.parseDateOozieTZ("2013-07-18T00:20Z"); Date retDate = CoordCommandUtils.getNextValidActionTimeForCronFrequency(actionTime, job); @@ -299,8 +298,124 @@ public class TestCoordCommandUtils extends XDataTestCase { assertEquals(expectedDate, retDate); } + @Test + public void testCoordAbsolute() throws Exception { + CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-dataset-absolute.xml", + CoordinatorJob.Status.RUNNING, false, true); + Path appPath = new Path(getFsTestCaseDir(), "coord"); + String actionXml = getCoordActionXml(appPath, "coord-dataset-absolute.xml"); + CoordinatorActionBean actionBean = createCoordinatorActionBean(job); + Configuration jobConf = new XConfiguration(new StringReader(job.getConf())); + Element eAction = createActionElement(actionXml); + jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)"); + jobConf.set("endInstance", "coord:current(2)"); + String output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), 1, + jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + List<?> elementList = ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)) + .getChildren(); + Element e1 = (Element) elementList.get(0); + Element e2 = (Element) elementList.get(1); + + // startInstance = coord:absolute(2009-08-20T01:00Z) which is current(0) + // and endInstance = coord:current(2). + assertEquals(e1.getChild("uris", e1.getNamespace()).getTextTrim(), + "hdfs:///tmp/workflows/2009/09/03;region=us#hdfs:///tmp/workflows/2009/08/27;" + + "region=us#hdfs:///tmp/workflows/2009/08/20;region=us"); + + // Test parameterized with startInstance = + // coord:absolute(2009-08-20T01:00Z) which is current (0) and + // endInstance = coord:current(2) + assertEquals(e2.getChild("uris", e1.getNamespace()).getTextTrim(), + "hdfs:///tmp/workflows/2009/09/03;region=us#hdfs:///tmp/workflows/2009/08/27;" + + "region=us#hdfs:///tmp/workflows/2009/08/20;region=us"); + + // Test when start instance < nominal time. 2009-08-20T01:00Z is + // current(-3) + + jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)"); + jobConf.set("endInstance", "coord:current(2)"); + eAction = createActionElement(actionXml); + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-09-10T01:00Z"), DateUtils.parseDateOozieTZ("2009-09-08T01:00Z"), 1, + jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + elementList = ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren(); + e1 = (Element) elementList.get(1); + assertEquals(e1.getChild("uris", e1.getNamespace()).getTextTrim(), + "hdfs:///tmp/workflows/2009/09/24;region=us#hdfs:///tmp/workflows/2009/09/17;region=us#" + + "hdfs:///tmp/workflows/2009/09/10;region=us#hdfs:///tmp/workflows/2009/09/03;region=us#" + + "hdfs:///tmp/workflows/2009/08/27;region=us#hdfs:///tmp/workflows/2009/08/20;region=us"); + + // // Test when start instance > nominal time. 2009-08-20T01:00Z is + // current(1) + jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)"); + jobConf.set("endInstance", "coord:current(2)"); + eAction = createActionElement(actionXml); + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-14T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-14T01:00Z"), 1, + jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + elementList = ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren(); + e1 = (Element) elementList.get(1); + assertEquals(e1.getChild("uris", e1.getNamespace()).getTextTrim(), + "hdfs:///tmp/workflows/2009/08/27;region=us#hdfs:///tmp/workflows/2009/08/20;region=us"); + + try { + // Test start instance > end instance. start = 2009-08-27T01:00Z + // which is current(3) + jobConf.set("startInstance", "coord:absolute(2009-08-27T01:00Z)"); + jobConf.set("endInstance", "coord:current(2)"); + eAction = createActionElement(actionXml); + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-06T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-16T01:00Z"), + 1, jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + fail("Should throw exception. Start-instance > end-instance "); + } + catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("start-instance should be equal or earlier than the end-instance")); + } + + try { + // Test start instance < initial instance. initial instance = + // 2009-08-06T01:00Z and start = 2009-07-01T01:00Z + jobConf.set("startInstance", "coord:absolute(2009-07-01T01:00Z)"); + jobConf.set("endInstance", "coord:current(2)"); + eAction = createActionElement(actionXml); + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-06T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-16T01:00Z"), + 1, jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + fail("Should throw exception. Start-instance > end-instance "); + } + catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("intial-instance should be equal or earlier than the start-instance")); + } + + try { + // Test Exception. Start-instance as absolute and end-instance as + // latest. + jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)"); + jobConf.set("endInstance", "coord:latest(2)"); + eAction = createActionElement(actionXml); + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), + 1, jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + fail("Should throw exception. Start-instance is absolute and end-instance is latest"); + } + catch (Exception e) { + assertTrue(e.getMessage().contains( + "Only start-instance as absolute and end-instance as current is supported")); + } + } + protected CoordinatorJobBean addRecordToCoordJobTable(CoordinatorJob.Status status, Date startTime, Date endTime, - String freq) throws Exception { + String freq) throws Exception { CoordinatorJobBean coordJob = createCoordJob(status, startTime, endTime, false, false, 0); coordJob.setStartTime(startTime); coordJob.setEndTime(endTime); http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java index 6550a60..34a428f 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java +++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java @@ -810,7 +810,7 @@ public class TestCoordELFunctions extends XTestCase { // TODO:Set hadoop properties eval.setVariable(CoordELFunctions.CONFIGURATION, conf); - String testDir = getTestCaseDir(); + getTestCaseDir(); ds.setUriTemplate(getTestCaseFileUri("${YEAR}/${MONTH}/${DAY}")); createTestCaseSubDir("2009/09/10/_SUCCESS".split("/")); // TODO: Create the directories @@ -874,7 +874,7 @@ public class TestCoordELFunctions extends XTestCase { // TODO:Set hadoop properties eval.setVariable(CoordELFunctions.CONFIGURATION, conf); - String testDir = getTestCaseDir(); + getTestCaseDir(); ds.setUriTemplate(getTestCaseFileUri("/${YEAR}/${MONTH}/${DAY}")); createTestCaseSubDir("2009/09/10/_SUCCESS".split("/")); createTestCaseSubDir("2009/09/11/_SUCCESS".split("/")); @@ -978,6 +978,34 @@ public class TestCoordELFunctions extends XTestCase { assertEquals("test_user", CoordELFunctions.evalAndWrap(eval, expr)); } + + public void testAbsoluteRange() throws Exception { + init("coord-action-create"); + ds = new SyncCoordDataset(); + ds.setFrequency(7); + ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-08-20T01:00Z")); + ds.setTimeUnit(TimeUnit.DAY); + ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles")); + ds.setName("test"); + ds.setUriTemplate("hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us"); + ds.setType("SYNC"); + ds.setDoneFlag(""); + appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-08-20T01:00Z")); + CoordELFunctions.configureEvaluator(eval, ds, appInst); + String expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"0\")}"; + assertEquals(CoordELFunctions.evalAndWrap(eval, expr), "2009-08-20T01:00Z"); + expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"1\")}"; + assertEquals(CoordELFunctions.evalAndWrap(eval, expr), "2009-08-27T01:00Z#2009-08-20T01:00Z"); + try { + expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"-1\")}"; + CoordELFunctions.evalAndWrap(eval, expr); + fail("start-instance is greater than the end-instance and there was no exception"); + } + catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("start-instance should be equal or earlier than the end-instance")); + } + } + /* * public void testDetach() throws Exception { Services.get().destroy(); } */ http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/core/src/test/resources/coord-dataset-absolute.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-dataset-absolute.xml b/core/src/test/resources/coord-dataset-absolute.xml new file mode 100644 index 0000000..8db7697 --- /dev/null +++ b/core/src/test/resources/coord-dataset-absolute.xml @@ -0,0 +1,48 @@ +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> + +<coordinator-app xmlns='uri:oozie:coordinator:0.2' name='NAME' + frequency="1" start='2009-08-20T01:00Z' end='2009-10-10T23:59Z' + timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> + <controls> + <timeout>10</timeout> + <concurrency>2</concurrency> + <execution>LIFO</execution> + </controls> + <input-events> + <data-in name="input" dataset="logs"> + <dataset name='logs' frequency='7' initial-instance='2009-08-06T01:00Z' + timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> + <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us + </uri-template> + </dataset> + <start-instance>${coord:absolute("2009-08-20T01:00Z")} + </start-instance> + <end-instance>${coord:current(2)}</end-instance> + </data-in> + <data-in name="input" dataset="test"> + <dataset name='test' frequency='7' initial-instance='2009-08-06T01:00Z' + timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> + <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us + </uri-template> + </dataset> + <start-instance>${startInstance}</start-instance> + <end-instance>${endInstance}</end-instance> + </data-in> + </input-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows/</app-path> + <configuration> + </configuration> + </workflow> + </action> +</coordinator-app> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 867bfcb..7cfbc0c 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -2094,13 +2094,62 @@ Then, the dataset instances for the input events for the coordinator action will hdfs://bar:8020/app/logs/2009/02/07 </verbatim> ----++++ 6.6.8. coord:version(int n) EL Function for Asynchronous Datasets + +---++++ 6.6.8. coord:absolute(String timeStamp) EL Function for Synchronous Datasets + +=${coord:absolute(String timeStamp)} = represents absolute dataset instance time. coord:absolute is only supported with range +where, start-instance is coord:absolute and end-instance is coord:current. Specifying a fixed date as the start instance is +useful if your processing needs to process all dataset instances from a specific instance to the current instance. + + +*%GREEN% Example: %ENDCOLOR%*: + +Coordinator application definition: + +<verbatim> +<coordinator-app name="app-coord" frequency="${coord:months(1)}" + start="2009-01-01T01:00Z" end="2009-12-31T24:00" timezone="UTC" + xmlns="uri:oozie:coordinator:0.4"> + <input-events> + <data-in name="input" dataset="logs"> + <dataset name='a' frequency='7' initial-instance="2009-01-01T01:00Z" + timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> + <uri-template>hdfs://bar:8020/app/logs/${YEAR}/${MONTH}/${DAY}/${HOUR} + </uri-template> + </dataset> + <start-instance>${coord:absolute("2009-01-01T01:00Z")}</start-instance> + <end-instance>${coord:current(0)}</end-instance> + </data-in> + </input-events> + <action> + <workflow> + ............. + </workflow> + </action> +</coordinator-app> +</verbatim> + +Then, the dataset instances for the input events for the coordinator action at first run will be: + +<verbatim> + hdfs://bar:8020/app/logs/2009/02/01 +<verbatim> + +The dataset instances for the input events for the coordinator action at second run will be: + +<verbatim> + hdfs://bar:8020/app/logs/2009/02/01 + hdfs://bar:8020/app/logs/2009/02/07 +<verbatim> + + +---++++ 6.6.9. coord:version(int n) EL Function for Asynchronous Datasets * TBD ----++++ 6.6.9. coord:latest(int n) EL Function for Asynchronous Datasets +---++++ 6.6.10. coord:latest(int n) EL Function for Asynchronous Datasets * TBD ----++++ 6.6.10. Dataset Instance Resolution for Instances Before the Initial Instance +---++++ 6.6.11. Dataset Instance Resolution for Instances Before the Initial Instance When defining input events that refer to dataset instances it may be possible that the resolution of instances is out of it lower bound. This is scenario is likely to happen when the instance resolution is very close to the initial-instance. This is useful for bootstrapping the application. http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/docs/src/site/twiki/WebServicesAPI.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki index 6be8fb9..50795b4 100644 --- a/docs/src/site/twiki/WebServicesAPI.twiki +++ b/docs/src/site/twiki/WebServicesAPI.twiki @@ -469,6 +469,7 @@ Content-Type: application/json;charset=UTF-8 } } } +</verbatim> ---+++ Job and Jobs End-Points http://git-wip-us.apache.org/repos/asf/oozie/blob/3a0a13a0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 46c39bb..6182190 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1504 Allow specifying a fixed instance as the start instance of a data-in (puru via rohini) OOZIE-1576 Add documentation for Oozie Sqoop CLI (bowenzhangusa via rkanter) OOZIE-1616 Add sharelib and launcherlib locations to the instrumentation info (rkanter) OOZIE-1647 oozie-setup.sh doesn't check exit code of java executions (alazarev via rkanter)
