Author: rkanter
Date: Fri Oct 18 22:40:56 2013
New Revision: 1533657
URL: http://svn.apache.org/r1533657
Log:
OOZIE-1578 Coordinator jobs with cron frequency should be invalidated upon
submission if nothing would be materialized during run time (bowenzhangusa via
rkanter)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
oozie/trunk/release-log.txt
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1533657&r1=1533656&r2=1533657&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
Fri Oct 18 22:40:56 2013
@@ -25,6 +25,7 @@ import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -306,11 +307,28 @@ public class CoordSubmitXCommand extends
/**
* Method that validates values in the definition for correctness.
Placeholder to add more.
*/
- private void validateCoordinatorJob() {
+ private void validateCoordinatorJob() throws Exception {
// check if startTime < endTime
if (!coordJob.getStartTime().before(coordJob.getEndTime())) {
throw new IllegalArgumentException("Coordinator Start Time must be
earlier than End Time.");
}
+
+ // Check if a coord job with cron frequency will materialize actions
+ try {
+ Integer.parseInt(coordJob.getFrequency());
+ } catch (NumberFormatException e) {
+ Date start = coordJob.getStartTime();
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(start);
+ cal.add(Calendar.MINUTE, -1);
+ start = cal.getTime();
+
+ Date nextTime =
CoordCommandUtils.getNextValidActionTimeForCronFrequency(start, coordJob);
+ if (!nextTime.before(coordJob.getEndTime())) {
+ throw new IllegalArgumentException("Coordinator job with
frequency '" +
+ coordJob.getFrequency() + "' materializes no actions
between start and end time.");
+ }
+ }
}
/*
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java?rev=1533657&r1=1533656&r2=1533657&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
Fri Oct 18 22:40:56 2013
@@ -175,6 +175,79 @@ public class TestCoordSubmitXCommand ext
}
}
+ public void testBasicSubmitWithUnMaterializableFrequency() throws
Exception {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ String appXml = "<coordinator-app name=\"NAME\" frequency=\"0/10 3 * *
*\" start=\"2010-02-01T01:00Z\" "
+ + "end=\"2010-02-01T02:00Z\" timezone=\"UTC\" "
+ + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> "
+ + "<execution>LIFO</execution> </controls> <datasets> "
+ + "<dataset name=\"a\" frequency=\"${coord:days(7)}\"
initial-instance=\"2010-02-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" +
getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> </dataset> "
+ + "<dataset name=\"local_a\" frequency=\"${coord:days(7)}\"
initial-instance=\"2010-02-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" +
getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> </dataset> "
+ + "</datasets> <input-events> "
+ + "<data-in name=\"A\" dataset=\"a\">
<instance>${coord:latest(0)}</instance> </data-in> "
+ + "</input-events> "
+ + "<output-events> <data-out name=\"LOCAL_A\"
dataset=\"local_a\"> "
+ + "<instance>${coord:current(-1)}</instance> </data-out>
</output-events> <action> <workflow> "
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ + "<configuration> <property> <name>inputA</name>
<value>${coord:dataIn('A')}</value> </property> "
+ + "<property> <name>inputB</name>
<value>${coord:dataOut('LOCAL_A')}</value> "
+ + "</property></configuration> </workflow> </action>
</coordinator-app>";
+ writeToFile(appXml, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH,
appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+
+ try {
+ sc.call();
+ fail("Expected to catch errors due to bad combination of frequency
and start and end time");
+ }
+ catch (CommandException e) {
+ assertEquals(sc.getJob().getStatus(), Job.Status.FAILED);
+ assertEquals(e.getErrorCode(), ErrorCode.E1003);
+ assertTrue(e.getMessage().contains("materializes no actions
between start and end time."));
+ }
+ }
+
+ public void testBasicSubmitWithValidFrequency() throws Exception {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ String appXml = "<coordinator-app name=\"NAME\" frequency=\"0/10 1 * *
*\" start=\"2010-02-01T01:00Z\" "
+ + "end=\"2010-02-01T02:00Z\" timezone=\"UTC\" "
+ + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> "
+ + "<execution>LIFO</execution> </controls> <datasets> "
+ + "<dataset name=\"a\" frequency=\"${coord:days(7)}\"
initial-instance=\"2009-02-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" +
getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> </dataset> "
+ + "<dataset name=\"local_a\" frequency=\"${coord:days(7)}\"
initial-instance=\"2009-02-01T01:00Z\" "
+ + "timezone=\"UTC\"> <uri-template>" +
getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}")
+ + "</uri-template> </dataset> "
+ + "</datasets> <input-events> "
+ + "<data-in name=\"A\" dataset=\"a\">
<instance>${coord:latest(0)}</instance> </data-in> "
+ + "</input-events> "
+ + "<output-events> <data-out name=\"LOCAL_A\"
dataset=\"local_a\"> "
+ + "<instance>${coord:current(-1)}</instance> </data-out>
</output-events> <action> <workflow> "
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ + "<configuration> <property> <name>inputA</name>
<value>${coord:dataIn('A')}</value> </property> "
+ + "<property> <name>inputB</name>
<value>${coord:dataOut('LOCAL_A')}</value> "
+ + "</property></configuration> </workflow> </action>
</coordinator-app>";
+ writeToFile(appXml, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH,
appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+
+ try {
+ sc.call();
+ }
+ catch (CommandException e) {
+ fail("Unexpected failure: " + e);
+ }
+ }
+
/**
* Testing for when user tries to submit a coordinator application having
data-in events
* that erroneously specify multiple input data instances inside a single
<instance> tag.
Modified: oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
URL:
http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki?rev=1533657&r1=1533656&r2=1533657&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki Fri Oct 18
22:40:56 2013
@@ -504,6 +504,10 @@ The legal characters and the names of mo
If a user specifies an invalid cron syntax to run something on Feb, 30th for
example: "0 10 30 2 *", the coordinator job
will not be created and an invalid coordinator frequency parse exception will
be thrown.
+If a user has a coordinator job that materializes no action during run time,
for example: frequency of "0 10 * * *" with
+start time of 2013-10-18T21:00Z and end time of 2013-10-18T22:00Z, the
coordinator job submission will be rejected and
+an invalid coordinator attribute exception will be thrown.
+
*%GREEN% Examples: %ENDCOLOR%*
| *Cron Expression* | *Meaning* |
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1533657&r1=1533656&r2=1533657&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Fri Oct 18 22:40:56 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1578 Coordinator jobs with cron frequency should be invalidated upon
submission if nothing would be materialized during run time (bowenzhangusa via
rkanter)
OOZIE-1577 Oozie coordinator job with identical start and end time remains
"RUNNING" forever (bowenzhangusa via rkanter)
OOZIE-1559 Fix missing fields from new SELECT queries and Recovery Service
picking up killed control nodes (ryota,mona via mona)
OOZIE-1569 Maintain backward incompatibility for running jobs before upgrade
(mona)
