Repository: oozie Updated Branches: refs/heads/master 647682940 -> 2a26670ea
OOZIE-2718 Improve -dryrun for bundles (zhengxb2005, asalamon74 via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2a26670e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2a26670e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2a26670e Branch: refs/heads/master Commit: 2a26670eae11c46086dcf683f0896bd278f116c5 Parents: 6476829 Author: Andras Piros <[email protected]> Authored: Mon Jul 30 15:35:18 2018 +0200 Committer: Andras Piros <[email protected]> Committed: Mon Jul 30 15:35:18 2018 +0200 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 13 +++-- .../command/bundle/BundleSubmitXCommand.java | 8 --- .../bundle/TestBundleSubmitXCommand.java | 14 +----- .../command/coord/TestCoordUpdateXCommand.java | 5 -- docs/src/site/twiki/DG_CommandLineTool.twiki | 53 +++++++++++++++++++- release-log.txt | 1 + 6 files changed, 63 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index c289971..9507a70 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -326,8 +326,8 @@ public class OozieCLI { Option debug = new Option(DEBUG_OPTION, false, "Use debug mode to see debugging statements on stdout"); Option rerun = new Option(RERUN_OPTION, true, "rerun a job (coordinator requires -action or -date, bundle requires -coordinator or -date)"); - Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2) or coordinator (since 2.0) job without" - + " actually executing it"); + Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2), a coordinator (since 2.0) " + + " or a bundle (since 5.1) job without actually executing it"); Option update = new Option(UPDATE_OPTION, true, "Update coord definition and properties"); Option showdiff = new Option(SHOWDIFF_OPTION, true, "Show diff of the new coord definition and properties with the existing one (default true)"); @@ -1020,7 +1020,14 @@ public class OozieCLI { String dryrunStr = wc.dryrun(getConfiguration(wc, commandLine)); if (dryrunStr.equals("OK")) { // workflow System.out.println("OK"); - } else { // coordinator + } + else if (dryrunStr.contains("<bundle-app")) { + // bundle + System.out.println("***Bundle job after parsing: ***"); + System.out.println(dryrunStr); + } + else { + // coordinator String[] dryrunStrs = dryrunStr.split("action for new instance"); int arraysize = dryrunStrs.length; System.out.println("***coordJob after parsing: ***"); http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java index f94e871..c06759e 100644 --- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java @@ -143,14 +143,6 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand { LogUtils.setLogInfo(bundleBean); if (dryrun) { - Date startTime = bundleBean.getStartTime(); - long startTimeMilli = startTime.getTime(); - long endTimeMilli = startTimeMilli + (3600 * 1000); - Date jobEndTime = bundleBean.getEndTime(); - Date endTime = new Date(endTimeMilli); - if (endTime.compareTo(jobEndTime) > 0) { - endTime = jobEndTime; - } jobId = bundleBean.getId(); LOG.info("[" + jobId + "]: Update status to PREP"); bundleBean.setStatus(Job.Status.PREP); http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java index 9c7a516..7c5f5b6 100644 --- a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java @@ -82,17 +82,14 @@ public class TestBundleSubmitXCommand extends XDataTestCase { jobConf.set(OozieClient.BUNDLE_APP_PATH, appPath.toString()); BundleSubmitXCommand command = new BundleSubmitXCommand(true, jobConf); - BundleJobBean bundleBean = (BundleJobBean) command.getJob(); - bundleBean.setStartTime(new Date()); - bundleBean.setEndTime(new Date()); command.call(); + assertEquals(Job.Status.PREP, command.getJob().getStatus()); // result includes bundle-submit-job.xml file instead of jobId since this is a dryRun mode String result = command.submit(); // bundle-submit-job.xml contains the Apache license but this result should not contain the comment block assertTrue("submit result should not contain <!-- ", !result.contains("<!--")); assertTrue("submit result should not contain --> ", !result.contains("-->")); - } public void testCoordJobNameParameterization() throws Exception { @@ -102,9 +99,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase { jobConf.set("coord1.starttime", "2009-02-01T00:00Z"); BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf); - final BundleJobBean bundleBean = (BundleJobBean) command.getJob(); - bundleBean.setStartTime(new Date()); - bundleBean.setEndTime(new Date()); final String jobId = command.call(); sleep(2000); new BundleStartXCommand(jobId).call(); @@ -129,9 +123,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase { jobConf.set("coordName2", "coord"); BundleSubmitXCommand command = new BundleSubmitXCommand(true, jobConf); - BundleJobBean bundleBean = (BundleJobBean) command.getJob(); - bundleBean.setStartTime(new Date()); - bundleBean.setEndTime(new Date()); try { command.call(); } @@ -148,9 +139,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase { jobConf.set("coord1.starttime", "2009-02-01T00:00Z"); BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf); - final BundleJobBean bundleBean = (BundleJobBean) command.getJob(); - bundleBean.setStartTime(new Date()); - bundleBean.setEndTime(new Date()); final String jobId = command.call(); sleep(2000); new BundleStartXCommand(jobId).call(); http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java index 1528a52..c002df6 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java @@ -25,14 +25,12 @@ import java.io.Reader; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.io.Writer; -import java.util.Date; import java.util.List; import java.util.regex.Matcher; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.oozie.BundleActionBean; -import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.XException; import org.apache.oozie.client.CoordinatorAction; @@ -371,9 +369,6 @@ public class TestCoordUpdateXCommand extends XDataTestCase { jobConf.set("coordName2", "coord2"); jobConf.set("isEnabled", "true"); BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf); - final BundleJobBean bundleBean = (BundleJobBean) command.getJob(); - bundleBean.setStartTime(new Date()); - bundleBean.setEndTime(new Date()); final String jobId = command.call(); sleep(2000); new BundleStartXCommand(jobId).call(); http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index d1d302a..8d33c50 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -53,8 +53,8 @@ oozie job <OPTIONS> : job operations -diff <arg> Show diff of the new coord definition and properties with the existing one (default true) -doas <arg> doAs user, impersonates as the specified user - -dryrun Dryrun a workflow (since 3.3.2) or coordinator (since 2.0) - job without actually executing it + -dryrun Dryrun a coordinator (since 2.0), a workflow (since 3.3.2) or a bundle (since 5.1) + without actually executing it. -failed re-runs the failed workflow actions of the coordinator actions (requires -rerun) -filter <arg> <key><comparator><value>[;<key><comparator><value>]* (All Coordinator actions satisfying the filters will be retrieved). @@ -890,6 +890,55 @@ specified path must be an HDFS path. If the workflow is accepted (i.e. Oozie is able to successfully read and parse it), it will return ="OK"=; otherwise, it will return an error message describing why it was rejected. +---+++ Dryrun of Bundle Job + +* This feature is only supported in Oozie 5.1 or later. + +Example: + +<verbatim> + +$ oozie job -oozie http://localhost:11000/oozie -dryrun -config job.properties +***Bundle job after parsing: *** +<bundle-app xmlns="uri:oozie:bundle:0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; name="My Bundle"> + <parameters> + <property> + <name>oozie.use.system.libpath</name> + <value>true</value> + </property> + </parameters> + <controls> + <kick-off-time>2017-02-14T00:13Z</kick-off-time> + </controls> + <coordinator name="My Coordinator-0"> + <app-path>hdfs://localhost:8020/user/hue/oozie/deployments/_admin_-oozie-24-1487060026.39</app-path> + <configuration> + <property> + <name>wf_application_path</name> + <value>hdfs://localhost:8020/user/hue/oozie/workspaces/hue-oozie-1486786607.01</value> + </property> + <property> + <name>start_date</name> + <value>2017-02-14T08:12Z</value> + </property> + <property> + <name>end_date</name> + <value>2017-02-21T08:12Z</value> + </property> + </configuration> + </coordinator> +</bundle-app> + +</verbatim> + +The =dryrun= option tests running a bundle job with given job properties and does not create the job. + +The parameters for the job must be provided in a file, either a Java Properties file (.properties) or a Hadoop XML +Configuration file (.xml). This file must be specified with the <code>-config</code> option. + +If the bundle job is accepted (i.e. Oozie is able to successfully read and parse it), it will return the parsed bundle job in xml; +otherwise, it will return an error message describing why it was rejected. + ---+++ Updating coordinator definition and properties Existing coordinator definition will be replaced by new definition. The refreshed coordinator would keep the same coordinator ID, state, and coordinator actions. All created coord action(including in WAITING) will use old configuration. http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index acd5949..fb0e020 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-2718 Improve -dryrun for bundles (zhengxb2005, asalamon74 via andras.piros) OOZIE-3156 amend Retry SSH action check when cannot connect to remote host (txsing, matijhs via andras.piros) OOZIE-3303 Oozie UI does not work after Jetty 9.3 upgrade (asalamon74 via gezapeti) OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti)
