Repository: oozie Updated Branches: refs/heads/master 9cea0e756 -> 7c78935b3
OOZIE-2130 Add EL Function for offsetting a date by a timezone amount including DST (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7c78935b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7c78935b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7c78935b Branch: refs/heads/master Commit: 7c78935b315895ad65b1fdd651825ccde5566e66 Parents: 9cea0e7 Author: Robert Kanter <[email protected]> Authored: Tue Apr 28 15:58:00 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Tue Apr 28 15:58:00 2015 -0700 ---------------------------------------------------------------------- .../apache/oozie/coord/CoordELFunctions.java | 37 ++++++++++--- .../java/org/apache/oozie/util/DateUtils.java | 12 +++++ core/src/main/resources/oozie-default.xml | 4 ++ .../oozie/coord/TestCoordELFunctions.java | 19 +++++-- .../site/twiki/CoordinatorFunctionalSpec.twiki | 56 ++++++++++++++++++-- release-log.txt | 1 + 6 files changed, 115 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java index 7f59186..0cf2c44 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java @@ -167,13 +167,13 @@ public class CoordELFunctions { } /** - * Returns the a date string while given a base date in 'strBaseDate', - * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH). + * Returns a date string that is offset from 'strBaseDate' by the amount specified. The unit can be one of + * DAY, MONTH, HOUR, MINUTE, MONTH. * - * @param strBaseDate -- base date - * @param offset -- any number - * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH - * @return date string + * @param strBaseDate The base date + * @param offset any number + * @param unit one of DAY, MONTH, HOUR, MINUTE, MONTH + * @return the offset date string * @throws Exception */ public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { @@ -189,6 +189,27 @@ public class CoordELFunctions { } /** + * Returns a date string that is offset from 'strBaseDate' by the difference from Oozie processing timezone to the given + * timezone. It will account for daylight saving time based on the given 'strBaseDate' and 'timezone'. + * + * @param strBaseDate The base date + * @param timezone + * @return the offset date string + * @throws Exception + */ + public static String ph2_coord_dateTzOffset(String strBaseDate, String timezone) throws Exception { + Calendar baseCalDate = DateUtils.getCalendar(strBaseDate); + StringBuilder buffer = new StringBuilder(); + baseCalDate.setTimeZone(DateUtils.getTimeZone(timezone)); + buffer.append(DateUtils.formatDate(baseCalDate)); + return buffer.toString(); + } + + public static String ph3_coord_dateTzOffset(String strBaseDate, String timezone) throws Exception{ + return ph2_coord_dateTzOffset(strBaseDate, timezone); + } + + /** * Determine the date-time in Oozie processing timezone of n-th future available dataset instance * from nominal Time but not beyond the instance specified as 'instance. * <p/> @@ -770,6 +791,10 @@ public class CoordELFunctions { return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit); } + public static String ph1_coord_dateTzOffset_echo(String n, String timezone) { + return echoUnResolved("dateTzOffset", n + " , " + timezone); + } + public static String ph1_coord_formatTime_echo(String dateTime, String format) { // Quote the dateTime value since it would contain a ':'. return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format); http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/java/org/apache/oozie/util/DateUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/DateUtils.java b/core/src/main/java/org/apache/oozie/util/DateUtils.java index ec9d0be..2b17a78 100644 --- a/core/src/main/java/org/apache/oozie/util/DateUtils.java +++ b/core/src/main/java/org/apache/oozie/util/DateUtils.java @@ -216,6 +216,18 @@ public class DateUtils { } /** + * Formats a {@link Calendar} as a string in ISO8601 format without adjusting its timezone. However, the mask will still + * ensure that the returned date is in the Oozie processing timezone. + * + * @param c {@link Calendar} to format. + * @return the ISO8601 string for the given date, <code>NULL</code> if the {@link Calendar} instance was + * <code>NULL</code> + */ + public static String formatDate(Calendar c) { + return (c != null) ? getISO8601DateFormat(c.getTimeZone(), ACTIVE_MASK).format(c.getTime()) : "NULL"; + } + + /** * This function returns number of hour in a day when given a Calendar with appropriate TZ. It consider DST to find * the number of hours. Generally it is 24. At some tZ, in one day of a year it is 23 and another day it is 25 * http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index ecc0d47..1048adc 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -946,6 +946,7 @@ coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap, coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actualTime_echo_wrap, coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo, coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo, @@ -1009,6 +1010,7 @@ coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed, coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actualTime_echo_wrap, coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo, coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo, @@ -1179,6 +1181,7 @@ coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime, coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actualTime, coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId, coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name, @@ -1240,6 +1243,7 @@ coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime, coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actualTime, coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset, + coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateTzOffset, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actionId, coord:name=org.apache.oozie.coord.CoordELFunctions#ph3_coord_name, http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java index 6d3bb48..6515b94 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java +++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java @@ -357,10 +357,8 @@ public class TestCoordELFunctions extends XTestCase { } public void testDateOffset() throws Exception { - init("coord-job-submit-data"); - String expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}"; init("coord-action-start"); - expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}"; + String expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", 2, \"DAY\")}"; assertEquals("2009-09-10T23:59Z", CoordELFunctions.evalAndWrap(eval, expr)); expr = "${coord:dateOffset(\"2009-09-08T23:59Z\", -1, \"DAY\")}"; @@ -370,6 +368,21 @@ public class TestCoordELFunctions extends XTestCase { assertEquals("2010-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr)); } + public void testDateTzOffset() throws Exception { + init("coord-action-start"); + // PDT is UTC - 7 + String expr = "${coord:dateTzOffset(\"2012-06-13T00:00Z\", \"America/Los_Angeles\")}"; //Summer + assertEquals("2012-06-12T17:00Z", CoordELFunctions.evalAndWrap(eval, expr)); + expr = "${coord:dateTzOffset(\"2012-06-13T00:00Z\", \"PST\")}"; + assertEquals("2012-06-12T17:00Z", CoordELFunctions.evalAndWrap(eval, expr)); + + // PST is UTC - 8 + expr = "${coord:dateTzOffset(\"2012-12-13T00:00Z\", \"America/Los_Angeles\")}"; //Winter + assertEquals("2012-12-12T16:00Z", CoordELFunctions.evalAndWrap(eval, expr)); + expr = "${coord:dateTzOffset(\"2012-12-13T00:00Z\", \"PST\")}"; + assertEquals("2012-12-12T16:00Z", CoordELFunctions.evalAndWrap(eval, expr)); + } + public void testCurrentRange() throws Exception { init("coord-action-create"); String expr = "${coord:currentRange(-1, 0)}"; http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 44a5be2..faff37f 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -3070,16 +3070,22 @@ This section describes the EL functions that could be used to parameterized both ---++++ 6.9.1. coord:dateOffset(String baseDate, int instance, String timeUnit) EL Function -The =${coord:dateOffset(String baseDate, int instance, String timeUnit)}= EL function calculates date based on the following equation : =newDate = baseDate + instance, * timeUnit= +The =${coord:dateOffset(String baseDate, int instance, String timeUnit)}= EL function calculates the date based on the following +equation : =newDate = baseDate + (instance * timeUnit)= +In other words, it offsets the =baseDate= by the amount specified by =instance= and =timeUnit=. -For example, if baseDate is '2009-01-01T00:00Z', instance is '2' and timeUnit is 'MONTH', the return date will be '2009-03-01T00:00Z'. If baseDate is '2009-01-01T00:00Z', instance is '1' and timeUnit is 'YEAR', the return date will be '2010-01-01T00:00Z'. +The =timeUnit= argument accepts one of 'DAY', 'MONTH', 'HOUR', 'MINUTE', 'MONTH' + +For example, if =baseDate= is '2009-01-01T00:00Z', =instance= is '2' and =timeUnit= is 'MONTH', the return date will be +'2009-03-01T00:00Z'. If =baseDate= is '2009-01-01T00:00Z', =instance= is '1' and =timeUnit= is 'YEAR', the return date will be +'2010-01-01T00:00Z'. *%GREEN% Example: %ENDCOLOR%*: <verbatim> <coordinator-app name="app-coord" frequency="${coord:days(1)}" - start="2009-01-01T24:00Z" end="2009-12-31T24:00Z" timezone="UTC" + start="2009-01-01T23:00Z" end="2009-12-31T23:00Z" timezone="UTC" xmlns="uri:oozie:coordinator:0.1"> ...... <action> @@ -3100,9 +3106,49 @@ For example, if baseDate is '2009-01-01T00:00Z', instance is '2' and timeUnit is </coordinator-app> </verbatim> -In this example, the 'nextInstance' will be '2009-01-02T24:00Z' for the first action. And the value of 'previousInstance' will be '2008-12-31T24:00Z' for the same instance. +In this example, the 'nextInstance' will be '2009-01-02T23:00Z' for the first action. And the value of 'previousInstance' will be +'2008-12-31T23:00Z' for the same instance. + +---++++ 6.9.2. coord:dateTzOffset(String baseDate, String timezone) EL Function + +The =${coord:dateTzOffset(String baseDate, String timezone)}= EL function calculates the date based on the following +equation : =newDate = baseDate + (Oozie procesing timezone - timezone)= +In other words, it offsets the =baseDate= by the difference from Oozie processing timezone to the given =timezone=. It will +account for daylight saving time based on the given =baseDate= and =timezone=. + +The =timezone= argument accepts any timezone or GMT offset that is returned by the +[[DG_CommandLineTool#Getting_a_list_of_time_zones]["info -timezones"]] command. For example, "America/Los_Angeles" or "PST". + +For example, if =baseDate= is '2012-06-13T00:00Z' and =timezone= is 'America/Los_Angeles', the return date will be +'2012-06-12T17:00Z'. But if =baseDate= is '2012-12-13T00:00Z', then the return date will be '2012-12-12T16:00Z'. The difference +in return dates occurs because the former occurs during Summer when DST is in effect (UTC-0700) and the latter occurs during Winter +when DST is no in effect (UTC-0800). + +*%GREEN% Example: %ENDCOLOR%*: + + +<verbatim> + <coordinator-app name="app-coord" frequency="${coord:days(1)}" + start="2009-01-01T24:00Z" end="2009-12-31T24:00Z" timezone="UTC" + xmlns="uri:oozie:coordinator:0.1"> + ...... + <action> + <workflow> + <app-path>hdfs://bar:8020/usr/joe/logsaggretor-wf</app-path> + <configuration> + <property> + <name>myDate</name> + <value>${coord:dateTzOffset(coord:nominalTime(), "America/Los_Angeles")}</value> + </property> + </configuration> + </workflow> + </action> + </coordinator-app> +</verbatim> + +In this example, the 'myDate' will be '2009-01-01T15:00Z' for the first action. ----++++ 6.9.2. coord:formatTime(String ts, String format) EL Function (since Oozie 2.3.2) +---++++ 6.9.3. coord:formatTime(String ts, String format) EL Function (since Oozie 2.3.2) The =${coord:formatTime(String timeStamp, String format)}= function allows transformation of the standard ISO8601 timestamp strings into other desired formats. http://git-wip-us.apache.org/repos/asf/oozie/blob/7c78935b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 48ac8c2..28051b7 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2130 Add EL Function for offsetting a date by a timezone amount including DST (rkanter) OOZIE-2199 Ooziedb.cmd and oozie-setup.ps1 are missing jars in lib/ for classpath on Windows (venkatnrangan via bzhang) OOZIE-2012 coordinator with an invalid cron frequency throws NPE after validation (bzhang) OOZIE-2129 Duplicate child jobs per instance (jaydeepvishwakarma via shwethags)
