Repository: oozie Updated Branches: refs/heads/master c729cab62 -> 5508ec7a6
OOZIE-1629 EL function in <timeout> is not evaluated properly (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5508ec7a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5508ec7a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5508ec7a Branch: refs/heads/master Commit: 5508ec7a660b00e0ea0a133d60ff49444c70ce1f Parents: c729cab Author: egashira <[email protected]> Authored: Thu Feb 20 16:15:09 2014 -0800 Committer: egashira <[email protected]> Committed: Thu Feb 20 16:15:09 2014 -0800 ---------------------------------------------------------------------- .../command/coord/CoordSubmitXCommand.java | 24 ++++++- core/src/main/resources/oozie-default.xml | 15 ++++- .../command/coord/TestCoordSubmitXCommand.java | 68 ++++++++++++++++++++ .../site/twiki/CoordinatorFunctionalSpec.twiki | 2 + release-log.txt | 1 + 5 files changed, 107 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java index 609b20c..712fe51 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java @@ -140,6 +140,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { private ELEvaluator evalInst = null; private ELEvaluator evalAction = null; private ELEvaluator evalSla = null; + private ELEvaluator evalTimeout = null; static { String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY, @@ -625,6 +626,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances"); evalSla = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-sla-submit"); evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start"); + evalTimeout = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-wait-timeout"); } /** @@ -683,8 +685,26 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { coordJob.setTimeZone(val); // controls - val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalFreq); - if (val == "") { + val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalTimeout); + if (val != null && val != "") { + int t = Integer.parseInt(val); + tmp = (evalTimeout.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalTimeout + .getVariable("timeunit")); + switch (tmp) { + case HOUR: + val = String.valueOf(t * 60); + break; + case DAY: + val = String.valueOf(t * 60 * 24); + break; + case MONTH: + val = String.valueOf(t * 60 * 24 * 30); + break; + default: + break; + } + } + else { val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 26e3f09..0fd1796 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -501,7 +501,7 @@ <!-- List of supported groups for ELService --> <property> <name>oozie.service.ELService.groups</name> - <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start</value> + <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout</value> <description>List of groups for different ELServices</description> </property> @@ -696,6 +696,19 @@ </property> <property> + <name>oozie.service.ELService.functions.coord-job-wait-timeout</name> + <value> + coord:days=org.apache.oozie.coord.CoordELFunctions#ph1_coord_days, + coord:months=org.apache.oozie.coord.CoordELFunctions#ph1_coord_months, + coord:hours=org.apache.oozie.coord.CoordELFunctions#ph1_coord_hours, + coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + </description> + </property> + + <property> <name>oozie.service.ELService.ext.functions.coord-job-submit-freq</name> <value> </value> http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java index dc289b4..88756f6 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java @@ -1151,4 +1151,72 @@ public class TestCoordSubmitXCommand extends XDataTestCase { } } } + + /** + * Test timeout setting + * + * @throws Exception + */ + public void testSubmitWithTimeout() throws Exception { + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + // timeout unit = DAY + String appXml1 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" " + + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" " + + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> " + + "<controls> <timeout>${coord:days(10)}</timeout> </controls> " + + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + + "</action> </coordinator-app>"; + writeToFile(appXml1, appPathFile); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + String jobId = sc.call(); + assertEquals(jobId.substring(jobId.length() - 2), "-C"); + CoordinatorJobBean job = checkCoordJobs(jobId); + assertEquals(job.getTimeout(), 14400); + // timeout unit = HOUR + String appXml2 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" " + + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" " + + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> " + + "<controls> <timeout>${coord:hours(10)}</timeout> </controls> " + + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + + "</action> </coordinator-app>"; + writeToFile(appXml2, appPathFile); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + sc = new CoordSubmitXCommand(conf); + jobId = sc.call(); + job = checkCoordJobs(jobId); + assertEquals(job.getTimeout(), 600); + // timeout unit = MINUTE + String appXml3 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" " + + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" " + + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> " + + "<controls> <timeout>${coord:minutes(10)}</timeout> </controls> " + + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + + "</action> </coordinator-app>"; + writeToFile(appXml3, appPathFile); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + sc = new CoordSubmitXCommand(conf); + jobId = sc.call(); + job = checkCoordJobs(jobId); + assertEquals(job.getTimeout(), 10); + // timeout unit = MONTH + String appXml4 = "<coordinator-app name=\"NAME\" frequency=\"${coord:months(1)}\" " + + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" " + + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> " + + "<controls> <timeout>${coord:months(1)}</timeout> </controls> " + + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + + "</action> </coordinator-app>"; + writeToFile(appXml4, appPathFile); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + sc = new CoordSubmitXCommand(conf); + jobId = sc.call(); + job = checkCoordJobs(jobId); + assertEquals(job.getTimeout(), 43200); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 7cfbc0c..44a3d54 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -208,6 +208,8 @@ Coordinator Frequencies can also be expressed using cron syntax. | =${coord:months(int n)}= | _variable_ | =${coord:months(1)}= --> minutes in a 1 full month from the current date | | =${cron syntax}= | _variable_ | =${0,10 15 * * 2-6}= --> a job that runs every weekday at 3:00pm and 3:10pm UTC time| +Note that, though =${coord:days(int n)}= and =${coord:months(int n)}= EL functions are used to calculate minutes precisely including variations due to daylight saving time for Frequency representation, when specified for coordinator timeout interval, one day is calculated as 24 hours and one month is calculated as 30 days for simplicity. + ---++++ 4.4.1. The coord:days(int n) and coord:endOfDays(int n) EL functions The =${coord:days(int n)}= and =${coord:endOfDays(int n)}= EL functions should be used to handle day based frequencies. http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 2248b40..40917d2 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1629 EL function in <timeout> is not evaluated properly (ryota) OOZIE-1618 dryrun should check variable substitution in workflow.xml (bowenzhangusa via rkanter) OOZIE-1681 Sqoop sharelib has no hsqldb jar version (Ostap via rkanter) OOZIE-1691 StackOverflowError in TimestampedMessageParser.parseNextLine() (puru via rkanter)
