Repository: oozie Updated Branches: refs/heads/master 2910636ee -> f713d6bd7
OOZIE-2368 coord:dateOffset and coord:dateTzOffset can't be used for coord initial-instance Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f713d6bd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f713d6bd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f713d6bd Branch: refs/heads/master Commit: f713d6bd72ab1c49740c5989cfde1d52ac42b34b Parents: 2910636 Author: Purshotam Shah <[email protected]> Authored: Tue Oct 27 13:48:42 2015 -0700 Committer: Purshotam Shah <[email protected]> Committed: Tue Oct 27 13:48:42 2015 -0700 ---------------------------------------------------------------------- .../command/coord/CoordSubmitXCommand.java | 5 +- .../apache/oozie/coord/CoordELEvaluator.java | 9 ++-- core/src/main/resources/oozie-default.xml | 23 +++++++-- .../command/coord/TestCoordSubmitXCommand.java | 50 ++++++++++++++++++++ .../oozie/coord/TestCoordELEvaluator.java | 17 ------- release-log.txt | 1 + 6 files changed, 78 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java index 587b983..d4d1c08 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java @@ -143,6 +143,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { private ELEvaluator evalAction = null; private ELEvaluator evalSla = null; private ELEvaluator evalTimeout = null; + private ELEvaluator evalInitialInstance = null; static { String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY, @@ -661,6 +662,8 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances"); evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start"); evalTimeout = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-wait-timeout"); + evalInitialInstance = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-initial-instance"); + } /** @@ -969,7 +972,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { .toString() : ((TimeUnit) evalFreq.getVariable("timeunit")).toString()); addAnAttribute("end_of_duration", dsElem, evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE .toString() : ((TimeUnit) evalFreq.getVariable("endOfDuration")).toString()); - val = resolveAttribute("initial-instance", dsElem, evalNofuncs); + val = resolveAttribute("initial-instance", dsElem, evalInitialInstance); ParamChecker.checkDateOozieTZ(val, "initial-instance"); checkInitialInstance(val); val = resolveAttribute("timezone", dsElem, evalNofuncs); http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java b/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java index d495dfa..8b2f456 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java @@ -73,7 +73,7 @@ public class CoordELEvaluator { Configuration conf) throws Exception { ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag); setConfigToEval(eval, conf); - SyncCoordDataset ds = getDSObject(eval, event); + SyncCoordDataset ds = getDSObject(event); CoordELFunctions.configureEvaluator(eval, ds, appInst); return eval; } @@ -104,7 +104,7 @@ public class CoordELEvaluator { throws Exception { ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start"); setConfigToEval(eval, conf); - SyncCoordDataset ds = getDSObject(eval, dEvent); + SyncCoordDataset ds = getDSObject(dEvent); SyncCoordAction appInst = new SyncCoordAction(); appInst.setNominalTime(nominalTime); appInst.setActualTime(actualTime); @@ -254,12 +254,11 @@ public class CoordELEvaluator { * @return * @throws Exception */ - private static SyncCoordDataset getDSObject(ELEvaluator eval, Element eData) throws Exception { + private static SyncCoordDataset getDSObject(Element eData) throws Exception { SyncCoordDataset ds = new SyncCoordDataset(); Element eDataset = eData.getChild("dataset", eData.getNamespace()); // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData)); - String expr = eDataset.getAttributeValue("initial-instance"); - Date initInstance = DateUtils.parseDateOozieTZ(eval.evaluate(expr, String.class)); + Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance")); ds.setInitInstance(initInstance); if (eDataset.getAttributeValue("frequency") != null) { int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency")); http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 649691d..faf3740 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -583,7 +583,7 @@ <!-- List of supported groups for ELService --> <property> <name>oozie.service.ELService.groups</name> - <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit</value> + <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit,coord-job-submit-initial-instance</value> <description>List of groups for different ELServices</description> </property> @@ -798,6 +798,18 @@ </property> <property> + <name>oozie.service.ELService.functions.coord-job-submit-initial-instance</name> + <value> + ${oozie.service.ELService.functions.coord-job-submit-nofuncs}, + coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset + </value> + <description> + EL functions for coord job submit initial instance, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + </description> + </property> + + <property> <name>oozie.service.ELService.ext.functions.coord-job-submit-freq</name> <value> </value> @@ -932,7 +944,9 @@ coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo, - hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf, + coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1104,7 +1118,6 @@ coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current, coord:currentRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_currentRange, coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset, - coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset, coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo, coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, @@ -1173,7 +1186,9 @@ coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range, - hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf, + coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java index e17d1b5..52eb9dd 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java @@ -28,12 +28,14 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.BundleJobBean; +import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.ErrorCode; import org.apache.oozie.client.Job; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; +import org.apache.oozie.executor.jpa.CoordActionQueryExecutor; import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor; import org.apache.oozie.executor.jpa.CoordJobQueryExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; @@ -1519,4 +1521,52 @@ public class TestCoordSubmitXCommand extends XDataTestCase { Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla")); assertEquals(SLAOperations.getTagElement(eSla, "max-duration"), "${5 * MINUTES}"); } + + public void testSubmitDateOffset() throws Exception { + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + String appXml = "<coordinator-app name=\"${NAME}\" frequency=\"${coord:days(1)}\" start=\"${startDate}\" " + + "end=\"${endDate}\" timezone=\"UTC\" " + + "xmlns=\"uri:oozie:coordinator:0.3\"> <controls> <timeout>10</timeout> <concurrency>2</concurrency> " + + "<execution>LIFO</execution> </controls> <datasets> " + + "<dataset name=\"a\" frequency=\"${coord:days(1)}\" " + + "initial-instance=\"${coord:dateOffset(startDate,-2,'DAY')}\" " + + "timezone=\"UTC\"> <uri-template>file:///tmp/coord/a/${YEAR}/${DAY}" + + "</uri-template> </dataset> " + + "<dataset name=\"b\" frequency=\"${coord:days(1)}\" initial-instance=\"${coord:dateTzOffset(startDate,'UTC')}\" " + + "timezone=\"UTC\"> <uri-template>file:///tmp/coord/b/${YEAR}/${DAY}</uri-template> " + + "<done-flag>${MY_DONE_FLAG}</done-flag> </dataset>" + + "</datasets> <input-events> " + + "<data-in name=\"A\" dataset=\"a\"> <start-instance>${coord:absolute(coord:dateOffset(startDate,-1,'DAY'))}" + + "</start-instance><end-instance>${coord:current(0)}</end-instance></data-in> " + + "<data-in name=\"B\" dataset=\"b\"> <start-instance>${coord:absolute(coord:dateTzOffset(startDate,'UTC'))}" + + "</start-instance><end-instance>${coord:current(0)}</end-instance></data-in> " + + "</input-events> " + + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> " + + "</workflow> </action> </coordinator-app>"; + writeToFile(appXml, appPathFile); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("MY_DONE_FLAG", "complete"); + conf.set("NAME", "test_app_name"); + conf.set("startDate", "2009-02-03T00:00Z"); + conf.set("endDate", "2009-03-03T00:00Z"); + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + String jobId = sc.call(); + CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get( + CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, jobId); + assertTrue(job.getJobXml() + .contains("dataset name=\"a\" frequency=\"1\" initial-instance=\"2009-02-01T00:00Z\"")); + assertTrue(job.getJobXml() + .contains("dataset name=\"b\" frequency=\"1\" initial-instance=\"2009-02-03T00:00Z\"")); + new CoordMaterializeTransitionXCommand(jobId, 3600).call(); + CoordinatorActionBean action = CoordActionQueryExecutor.getInstance().get( + CoordActionQueryExecutor.CoordActionQuery.GET_COORD_ACTION, jobId+"@1"); + action.getActionXml(); + assertTrue(action.getActionXml() + .contains("tmp/coord/a/2009/02")); + assertTrue(action.getActionXml() + .contains("tmp/coord/b/2009/03")); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java index b67e77e..a429665 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java +++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java @@ -171,23 +171,6 @@ public class TestCoordELEvaluator extends XTestCase { assertEquals("2009-09-08T00:00Z", eval.evaluate(expr, String.class)); } - public void testCreateInstancesELEvaluatorWithDateOffset() throws Exception { - SyncCoordAction appInst = new SyncCoordAction(); - appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-09-08T01:00Z")); - appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z")); - appInst.setTimeUnit(TimeUnit.MINUTE); - String dataEvntXML = "<data-in name=\"A\" dataset=\"a\"><uris>file:///tmp/coord/US/2009/1/30|file:///tmp/coord/" - + "US/2009/1/31</uris>"; - dataEvntXML += "<dataset name=\"a\" frequency=\"1440\" initial-instance=\"${coord:dateOffset('2009-09-08T23:59Z'" - + ", 2, 'MINUTE')}\" freq_timeunit=\"MINUTE\" timezone=\"UTC\" end_of_duration=\"NONE\">"; - dataEvntXML += "<uri-template>file:///tmp/coord/US/${YEAR}/${MONTH}/${DAY}</uri-template></dataset></data-in>"; - Element event = XmlUtils.parseXml(dataEvntXML); - Configuration conf = new XConfiguration(new StringReader(getConfString())); - ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf); - SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(CoordELFunctions.DATASET); - assertEquals("2009-09-09T00:01Z", DateUtils.formatDateOozieTZ(ds.getInitInstance())); - } - public void testCreateLazyEvaluator() throws Exception { // Configuration conf = new // XConfiguration(IOUtils.getResourceAsReader("org/apache/oozie/coord/conf.xml", http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7f687c6..9a0b9c5 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2368 coord:dateOffset and coord:dateTzOffset can't be used for coord initial-instance (puru) OOZIE-2369 coord:offset doesn't resolve correctly (puru) OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter) OOZIE-2314 Unable to kill old instance child job by workflow or coord rerun by Launcher (jaydeepvishwakarma via rkanter)
