Repository: oozie Updated Branches: refs/heads/master d7b2a28e5 -> c2f1edcae
OOZIE-2551 Feature request: epoch timestamp generation Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c2f1edca Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c2f1edca Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c2f1edca Branch: refs/heads/master Commit: c2f1edcae1cf1f94b2b90cdaf87c7ef8240b6db4 Parents: d7b2a28 Author: Purshotam Shah <[email protected]> Authored: Fri Jun 3 12:22:33 2016 -0700 Committer: Purshotam Shah <[email protected]> Committed: Fri Jun 3 12:22:33 2016 -0700 ---------------------------------------------------------------------- .../apache/oozie/coord/CoordELFunctions.java | 24 +++++++++++++++++ .../java/org/apache/oozie/util/DateUtils.java | 18 +++++++++++++ core/src/main/resources/oozie-default.xml | 7 +++++ .../oozie/coord/TestCoordELFunctions.java | 27 ++++++++++++++++++++ .../site/twiki/CoordinatorFunctionalSpec.twiki | 11 ++++++++ release-log.txt | 1 + 6 files changed, 88 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java index 5bb8be6..22eb1c3 100644 --- a/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java +++ b/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java @@ -439,6 +439,25 @@ public class CoordELFunctions { } /** + * Convert from standard date-time formatting to a Unix epoch time. + * <p/> + * @param dateTimeStr - A timestamp in standard (ISO8601) format. + * @param millis - "true" to include millis; otherwise will only include seconds + * @return coordinator action creation or materialization date time + * @throws Exception if unable to format the Date object to String + */ + public static String ph2_coord_epochTime(String dateTimeStr, String millis) + throws Exception { + Date dateTime = DateUtils.parseDateOozieTZ(dateTimeStr); + return DateUtils.formatDateEpoch(dateTime, Boolean.valueOf(millis)); + } + + public static String ph3_coord_epochTime(String dateTimeStr, String millis) + throws Exception { + return ph2_coord_epochTime(dateTimeStr, millis); + } + + /** * Return Action Id. <p> * * @return coordinator action Id @@ -819,6 +838,11 @@ public class CoordELFunctions { return echoUnResolved("dateTzOffset", n + " , " + timezone); } + public static String ph1_coord_epochTime_echo(String dateTime, String millis) { + // Quote the dateTime value since it would contain a ':'. + return echoUnResolved("epochTime", "'"+dateTime+"'" + " , " + millis); + } + public static String ph1_coord_formatTime_echo(String dateTime, String format) { // Quote the dateTime value since it would contain a ':'. return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format); http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/core/src/main/java/org/apache/oozie/util/DateUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/DateUtils.java b/core/src/main/java/org/apache/oozie/util/DateUtils.java index e20aca3..3caf0a2 100644 --- a/core/src/main/java/org/apache/oozie/util/DateUtils.java +++ b/core/src/main/java/org/apache/oozie/util/DateUtils.java @@ -205,6 +205,24 @@ public class DateUtils { } /** + * Formats a {@link Date} as a string containing the seconds (or millis) since the Unix epoch (Jan 1, 1970). + * <p/> + * The format mask must be a {@link SimpleDateFormat} valid format mask + * + * @param d {@link Date} to format. + * @param millis true to include milliseconds + * @return the number of seconds or millis between the given date and Jan 1, 1970, + * <code>NULL</code> if the {@link Date} instance was <code>NULL</code> + */ + public static String formatDateEpoch(Date d, Boolean millis) { + if (d == null) { + return "NULL"; + } else { + return Long.toString(millis ? d.getTime() : d.getTime() / 1000); + } + } + + /** * Formats a {@link Calendar} as a string in ISO8601 format using Oozie processing timezone. * * @param c {@link Calendar} to format. http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 8c268d6..6c2f7d8 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -949,6 +949,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:future=org.apache.oozie.coord.CoordELFunctions#ph1_coord_future_echo, coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_futureRange_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_epochTime_echo, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo, @@ -1003,6 +1004,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_epochTime_echo, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo, coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, @@ -1072,6 +1074,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_epochTime_echo, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo, coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, @@ -1133,6 +1136,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId, coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_epochTime, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, @@ -1190,6 +1194,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_epochTime, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, @@ -1245,6 +1250,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset, coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_epochTime, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId, coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, @@ -1307,6 +1313,7 @@ will be the requeue interval for the actions which are waiting for a long time w coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset, coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateTzOffset, coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime, + coord:epochTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_epochTime, coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actionId, coord:name=org.apache.oozie.coord.CoordELFunctions#ph3_coord_name, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java index 6515b94..fb7e030 100644 --- a/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java +++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELFunctions.java @@ -889,6 +889,33 @@ public class TestCoordELFunctions extends XTestCase { assertEquals(expr3_eval, CoordELFunctions.evalAndWrap(eval, expr3)); } + public void testEpochTime() throws Exception { + String expr1 = "${coord:epochTime(\"2009-09-08T23:59Z\", \"false\")}"; + String expr2 = "${coord:epochTime(\"2009-09-08T23:59Z\", \"true\")}"; + init("coord-action-create"); + assertEquals("1252454340", CoordELFunctions.evalAndWrap(eval, expr1)); + assertEquals("1252454340000", CoordELFunctions.evalAndWrap(eval, expr2)); + init("coord-action-create-inst"); + assertEquals("1252454340", CoordELFunctions.evalAndWrap(eval, expr1)); + assertEquals("1252454340000", CoordELFunctions.evalAndWrap(eval, expr2)); + init("coord-action-start"); + assertEquals("1252454340", CoordELFunctions.evalAndWrap(eval, expr1)); + assertEquals("1252454340000", CoordELFunctions.evalAndWrap(eval, expr2)); + + String utcDate = "2009-09-08T23:59Z"; + String expr3 = "${coord:epochTime(date, \"true\")}"; + String expr3_eval = "${coord:epochTime('" + utcDate + "' , " + "true)}"; + init("coord-job-submit-instances"); + eval.setVariable("date", utcDate); + assertEquals(expr3_eval, CoordELFunctions.evalAndWrap(eval, expr3)); + init("coord-job-submit-data"); + eval.setVariable("date", utcDate); + assertEquals(expr3_eval, CoordELFunctions.evalAndWrap(eval, expr3)); + init("coord-sla-submit"); + eval.setVariable("date", utcDate); + assertEquals(expr3_eval, CoordELFunctions.evalAndWrap(eval, expr3)); + } + public void testFuture() throws Exception { init("coord-job-submit-instances"); String expr = "${coord:future(1, 20)}"; http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 75b7354..05ad5be 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -3156,6 +3156,17 @@ The format string should be in Java's [[http://download.oracle.com/javase/6/docs For example, if timeStamp is '2009-01-01T00:00Z' and format is 'yyyy', the returned date string will be '2009'. +---++++ 6.9.4. coord:epochTime(String ts, String millis) EL Function (since Oozie 4.3) + +The =${coord:epochTime(String timeStamp, String millis)}= function allows transformation of the standard ISO8601 timestamp +strings into Unix epoch time (seconds or milliseconds since January 1, 1970). + +If millis is 'false', the returned time string will be the number of seconds since the epoch. If 'true', the returned time string +will be the number of milliseconds since the epoch. + +For example, if timeStamp is '2009-01-01T00:00Z' and millis is 'false', the returned date string will be '1230768000'. If millis +is 'true', the returned date string will be '1230768000000'. + ---++ 7. Handling Timezones and Daylight Saving Time As mentioned in section #4.1.1 'Timezones and Daylight-Saving', the coordinator engine works exclusively in UTC, and dataset and application definitions are always expressed in UTC. http://git-wip-us.apache.org/repos/asf/oozie/blob/c2f1edca/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index af207dc..1f13c26 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru) OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru) OOZIE-2447 Illegal character 0x0 oozie client (satishsaley via puru) OOZIE-2548 Flaky test TestZKLocksService.testLockReaper (pbacsko via puru)
