Repository: oozie Updated Branches: refs/heads/master 67563df53 -> 2910636ee
OOZIE-2369 coord:offset doesn't resolve correctly Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2910636e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2910636e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2910636e Branch: refs/heads/master Commit: 2910636eeb5249a7fe9ead11bc45a106a0097c8c Parents: 67563df Author: Purshotam Shah <[email protected]> Authored: Tue Oct 27 12:40:46 2015 -0700 Committer: Purshotam Shah <[email protected]> Committed: Tue Oct 27 12:40:46 2015 -0700 ---------------------------------------------------------------------- .../oozie/command/coord/CoordCommandUtils.java | 3 +- .../apache/oozie/coord/CoordELFunctions.java | 2 +- .../command/coord/TestCoordCommandUtils.java | 29 +++++++++++++++++--- .../src/test/resources/coord-dataset-offset.xml | 6 ++-- release-log.txt | 1 + 5 files changed, 32 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2910636e/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java index 7847c1f..131368a 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java @@ -262,8 +262,9 @@ public class CoordCommandUtils { if (startCal != null && endCal != null) { List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval); for (int i = expandedFreqs.size() - 1; i >= 0; i--) { + //we need to use DS timeout, bcz expandOffsetTimes will expand offset in Freqs in DS timeunit String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i) - + ", \"" + startRestArg + "\")}", appInst, conf, eval); + + ", \"" + CoordELFunctions.getDSTimeUnit(eval) + "\")}", appInst, conf, eval); if (matInstance == null || matInstance.length() == 0) { // Earlier than dataset's initial instance break; http://git-wip-us.apache.org/repos/asf/oozie/blob/2910636e/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java index 4828a23..5d23866 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java @@ -1463,7 +1463,7 @@ public class CoordELFunctions { /** * @return dataset TimeUnit */ - private static TimeUnit getDSTimeUnit(ELEvaluator eval) { + public static TimeUnit getDSTimeUnit(ELEvaluator eval) { SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); if (ds == null) { throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); http://git-wip-us.apache.org/repos/asf/oozie/blob/2910636e/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java index 3477894..652d792 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordCommandUtils.java @@ -305,21 +305,42 @@ public class TestCoordCommandUtils extends XDataTestCase { CoordinatorJob.Status.RUNNING, false, true); Path appPath = new Path(getFsTestCaseDir(), "coord"); String actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml"); + actionXml = actionXml.replace("-unit-", "DAY"); + actionXml = actionXml.replace("-frequency-", "1"); CoordinatorActionBean actionBean = createCoordinatorActionBean(job); Configuration jobConf = new XConfiguration(new StringReader(job.getConf())); Element eAction = createActionElement(actionXml); jobConf.set("startInstance", "coord:offset(-4,DAY)"); jobConf.set("endInstance", "coord:offset(0,DAY)"); String output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, - DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), 1, + DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), 1, jobConf, actionBean); eAction = XmlUtils.parseXml(output); Element e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)) .getChildren().get(0); assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), - "hdfs:///tmp/workflows/2009/08/20;region=us#hdfs:///tmp/workflows/2009/08/19;region=us#" - + "hdfs:///tmp/workflows/2009/08/18;region=us#hdfs:///tmp/workflows/2009/08/17;" - + "region=us#hdfs:///tmp/workflows/2009/08/16;region=us"); + "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/19/01;region=us#" + + "hdfs:///tmp/workflows/2009/08/18/01;region=us#hdfs:///tmp/workflows/2009/08/17/01;" + + "region=us#hdfs:///tmp/workflows/2009/08/16/01;region=us"); + + jobConf.set("startInstance", "coord:offset(-4,HOUR)"); + jobConf.set("endInstance", "coord:offset(0,HOUR)"); + actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml"); + actionXml = actionXml.replace("-unit-", "MINUTE"); + actionXml = actionXml.replace("-frequency-", "60"); + eAction = createActionElement(actionXml); + + output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, + DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), 1, + jobConf, actionBean); + eAction = XmlUtils.parseXml(output); + e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)) + .getChildren().get(0); + assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), + "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/20/00;region=us#" + + "hdfs:///tmp/workflows/2009/08/19/23;region=us#hdfs:///tmp/workflows/2009/08/19/22;region=us#" + + "hdfs:///tmp/workflows/2009/08/19/21;region=us"); + } @Test http://git-wip-us.apache.org/repos/asf/oozie/blob/2910636e/core/src/test/resources/coord-dataset-offset.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-dataset-offset.xml b/core/src/test/resources/coord-dataset-offset.xml index 4a7a7a3..5dfbb1b 100644 --- a/core/src/test/resources/coord-dataset-offset.xml +++ b/core/src/test/resources/coord-dataset-offset.xml @@ -19,9 +19,9 @@ </controls> <input-events> <data-in name="input" dataset="test"> - <dataset name='test' frequency='1' initial-instance='2009-08-06T01:00Z' - timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'> - <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us + <dataset name='test' frequency='-frequency-' initial-instance='2009-08-06T01:00Z' + timezone='UTC' freq_timeunit='-unit-' end_of_duration='NONE'> + <uri-template>hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY}/${HOUR};region=us </uri-template> </dataset> <start-instance>${startInstance}</start-instance> http://git-wip-us.apache.org/repos/asf/oozie/blob/2910636e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 7cbbcb5..7f687c6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2369 coord:offset doesn't resolve correctly (puru) OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter) OOZIE-2314 Unable to kill old instance child job by workflow or coord rerun by Launcher (jaydeepvishwakarma via rkanter) OOZIE-2386 org.apache.oozie.util.TestXConfiguration.testSubstituteVar is flakey (rkanter)
