Repository: oozie Updated Branches: refs/heads/master cc02d91d9 -> d1c475c32
OOZIE-2092 Provide option to supply config to workflow during rerun of coordinator (jaydeepvishwakarma via shwethags) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/d1c475c3 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/d1c475c3 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/d1c475c3 Branch: refs/heads/master Commit: d1c475c32ac647649cdd7258e5fd7a0d9f3764f4 Parents: cc02d91 Author: shwethags <[email protected]> Authored: Tue Jan 6 11:36:11 2015 +0530 Committer: shwethags <[email protected]> Committed: Tue Jan 6 11:36:11 2015 +0530 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 13 ++- .../org/apache/oozie/client/OozieClient.java | 11 ++- .../org/apache/oozie/CoordinatorEngine.java | 4 +- .../org/apache/oozie/LocalOozieClientCoord.java | 15 ++-- .../command/bundle/BundleRerunXCommand.java | 5 +- .../command/coord/CoordActionStartXCommand.java | 21 ++++- .../oozie/command/coord/CoordRerunXCommand.java | 29 +++++-- .../org/apache/oozie/servlet/V1JobServlet.java | 2 +- .../command/coord/TestCoordRerunXCommand.java | 84 +++++++++++++++++--- .../apache/oozie/event/TestEventGeneration.java | 4 +- .../servlet/MockCoordinatorEngineService.java | 2 +- .../oozie/sla/TestSLAEventGeneration.java | 2 +- .../site/twiki/CoordinatorFunctionalSpec.twiki | 3 +- docs/src/site/twiki/DG_CommandLineTool.twiki | 2 +- docs/src/site/twiki/DG_CoordinatorRerun.twiki | 3 +- release-log.txt | 1 + 16 files changed, 156 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index 0d3b758..bc8fa50 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -1032,6 +1032,7 @@ public class OozieCLI { String rerunType = null; boolean refresh = false; boolean noCleanup = false; + boolean failed = false; if (options.contains(DATE_OPTION) && options.contains(ACTION_OPTION)) { throw new OozieCLIException("Invalid options provided for rerun: either" + DATE_OPTION + " or " + ACTION_OPTION + " expected. Don't use both at the same time."); @@ -1054,11 +1055,17 @@ public class OozieCLI { if (options.contains(RERUN_NOCLEANUP_OPTION)) { noCleanup = true; } + + Properties props = null; + if(isConfigurationSpecified(wc, commandLine)) { + props = getConfiguration(wc, commandLine); + } + if (options.contains(RERUN_FAILED_OPTION)) { - printCoordActions(wc.reRunCoord(coordJobId, rerunType, scope, refresh, noCleanup, true)); - } else { - printCoordActions(wc.reRunCoord(coordJobId, rerunType, scope, refresh, noCleanup)); + failed = true; } + + printCoordActions(wc.reRunCoord(coordJobId, rerunType, scope, refresh, noCleanup, failed, props)); } } else if (options.contains(INFO_OPTION)) { http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index b1bedc6..800e871 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -1475,13 +1475,16 @@ public class OozieClient { } } private class CoordRerun extends ClientCallable<List<CoordinatorAction>> { + private final Properties conf; - CoordRerun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup, boolean failed) { + CoordRerun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup, boolean failed, + Properties conf) { super("PUT", RestConstants.JOB, notEmpty(jobId, "jobId"), prepareParams(RestConstants.ACTION_PARAM, RestConstants.JOB_COORD_ACTION_RERUN, RestConstants.JOB_COORD_RANGE_TYPE_PARAM, rerunType, RestConstants.JOB_COORD_SCOPE_PARAM, scope, RestConstants.JOB_COORD_RERUN_REFRESH_PARAM, Boolean.toString(refresh), RestConstants.JOB_COORD_RERUN_NOCLEANUP_PARAM, Boolean .toString(noCleanup), RestConstants.JOB_COORD_RERUN_FAILED_PARAM, Boolean.toString(failed))); + this.conf = conf; } @Override @@ -1535,7 +1538,7 @@ public class OozieClient { */ public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup) throws OozieClientException { - return new CoordRerun(jobId, rerunType, scope, refresh, noCleanup, false).call(); + return new CoordRerun(jobId, rerunType, scope, refresh, noCleanup, false, null).call(); } /** @@ -1550,8 +1553,8 @@ public class OozieClient { * @throws OozieClientException */ public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh, - boolean noCleanup, boolean failed) throws OozieClientException { - return new CoordRerun(jobId, rerunType, scope, refresh, noCleanup, failed).call(); + boolean noCleanup, boolean failed, Properties props) throws OozieClientException { + return new CoordRerun(jobId, rerunType, scope, refresh, noCleanup, failed, props).call(); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/CoordinatorEngine.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java index cce3f84..3406c6f 100644 --- a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java +++ b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java @@ -261,11 +261,11 @@ public class CoordinatorEngine extends BaseEngine { * @throws BaseEngineException */ public CoordinatorActionInfo reRun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup, - boolean failed) + boolean failed, Configuration conf) throws BaseEngineException { try { return new CoordRerunXCommand(jobId, rerunType, scope, refresh, - noCleanup, failed).call(); + noCleanup, failed, conf).call(); } catch (CommandException ex) { throw new BaseEngineException(ex); http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java b/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java index 2b25561..0932e6c 100644 --- a/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java +++ b/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java @@ -253,7 +253,7 @@ public class LocalOozieClientCoord extends OozieClient { @Override public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup) throws OozieClientException { - return getCoordinatorActions(jobId, rerunType, scope, refresh, noCleanup, false); + return getCoordinatorActions(jobId, rerunType, scope, refresh, noCleanup, false, null); } /** @@ -266,23 +266,28 @@ public class LocalOozieClientCoord extends OozieClient { * @param refresh true if -refresh is given in command option * @param noCleanup true if -nocleanup is given in command option * @param failed true if -failed is given in command option + * @param conf configuration information for the rerun * @throws OozieClientException */ @Override public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh, - boolean noCleanup, boolean failed) throws OozieClientException { - return getCoordinatorActions(jobId, rerunType, scope, refresh, noCleanup, failed); + boolean noCleanup, boolean failed, Properties conf ) throws OozieClientException { + return getCoordinatorActions(jobId, rerunType, scope, refresh, noCleanup, failed, conf); } private List<CoordinatorAction> getCoordinatorActions(String jobId, String rerunType, String scope, boolean refresh, - boolean noCleanup, boolean failed) throws OozieClientException { + boolean noCleanup, boolean failed, Properties prop) throws OozieClientException { try { + XConfiguration conf = null; + if (prop != null) { + conf = new XConfiguration(prop); + } if (!(rerunType.equals(RestConstants.JOB_COORD_SCOPE_DATE) || rerunType .equals(RestConstants.JOB_COORD_SCOPE_ACTION))) { throw new CommandException(ErrorCode.E1018, "date or action expected."); } CoordinatorActionInfo coordInfo = coordEngine.reRun(jobId, rerunType, scope, Boolean.valueOf(refresh), - Boolean.valueOf(noCleanup), Boolean.valueOf(failed)); + Boolean.valueOf(noCleanup), Boolean.valueOf(failed), conf); List<CoordinatorActionBean> actionBeans; if (coordInfo != null) { actionBeans = coordInfo.getCoordActions(); http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/command/bundle/BundleRerunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleRerunXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleRerunXCommand.java index 8de2c70..90d72e3 100644 --- a/core/src/main/java/org/apache/oozie/command/bundle/BundleRerunXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleRerunXCommand.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.oozie.BundleActionBean; import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorJobBean; @@ -139,7 +140,7 @@ public class BundleRerunXCommand extends RerunTransitionXCommand<Void> { LOG.debug("Queuing rerun range [" + rerunDateScope + "] for coord id " + coordId + " of bundle " + bundleJob.getId()); queue(new CoordRerunXCommand(coordId, RestConstants.JOB_COORD_SCOPE_DATE, rerunDateScope, refresh, - noCleanup, false)); + noCleanup, false, null)); updateBundleAction(coordNameToBAMapping.get(coordName)); isUpdateActionDone = true; } @@ -159,7 +160,7 @@ public class BundleRerunXCommand extends RerunTransitionXCommand<Void> { LOG.debug("Queuing rerun range [" + dateScope + "] for coord id " + action.getCoordId() + " of bundle " + bundleJob.getId()); queue(new CoordRerunXCommand(action.getCoordId(), RestConstants.JOB_COORD_SCOPE_DATE, dateScope, - refresh, noCleanup, false)); + refresh, noCleanup, false, null)); updateBundleAction(action); isUpdateActionDone = true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/command/coord/CoordActionStartXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordActionStartXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordActionStartXCommand.java index 2223b13..98f356e 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordActionStartXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordActionStartXCommand.java @@ -120,6 +120,7 @@ public class CoordActionStartXCommand extends CoordinatorXCommand<Void> { Configuration runConf = null; try { runConf = new XConfiguration(new StringReader(createdConf)); + } catch (IOException e1) { log.warn("Configuration parse error in:" + createdConf); @@ -151,9 +152,25 @@ public class CoordActionStartXCommand extends CoordinatorXCommand<Void> { // new property called 'oozie.wf.application.path' // WF Engine requires the path to the workflow.xml to be saved under // this property name - String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow", - workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue(); + String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()) + .getChild("workflow", workflowProperties.getNamespace()).getChild("app-path", + workflowProperties.getNamespace()).getValue(); + + // Copying application path in runconf. runConf.set("oozie.wf.application.path", appPath); + + // Step 4: Extract the runconf and copy the rerun config to runconf. + if (runConf.get(CoordRerunXCommand.RERUN_CONF) != null) { + Configuration rerunConf = null; + try { + rerunConf = new XConfiguration(new StringReader(runConf.get(CoordRerunXCommand.RERUN_CONF))); + XConfiguration.copy(rerunConf, runConf); + } catch (IOException e) { + log.warn("Configuration parse error in:" + rerunConf); + throw new CommandException(ErrorCode.E1005, e.getMessage(), e); + } + runConf.unset(CoordRerunXCommand.RERUN_CONF); + } return runConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/command/coord/CoordRerunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordRerunXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordRerunXCommand.java index cbace93..050662e 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordRerunXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordRerunXCommand.java @@ -84,6 +84,7 @@ import org.jdom.JDOMException; @SuppressWarnings("deprecation") public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActionInfo> { + public static final String RERUN_CONF = "rerunConf"; private String rerunType; private String scope; private boolean refresh; @@ -91,6 +92,7 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio private CoordinatorJobBean coordJob = null; protected boolean prevPending; private boolean failed; + private Configuration actionRunConf; /** * The constructor for class {@link CoordRerunXCommand} @@ -100,9 +102,11 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio * @param scope the rerun scope for given rerunType separated by "," * @param refresh true if user wants to refresh input/output dataset urls * @param noCleanup false if user wants to cleanup output events for given rerun actions + * @param failed true if user wants to rerun only failed nodes + * @param actionRunConf configuration values for actions */ public CoordRerunXCommand(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup, - boolean failed) { + boolean failed, Configuration actionRunConf) { super("coord_rerun", "coord_rerun", 1); this.jobId = ParamChecker.notEmpty(jobId, "jobId"); this.rerunType = ParamChecker.notEmpty(rerunType, "rerunType"); @@ -110,12 +114,13 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio this.refresh = refresh; this.noCleanup = noCleanup; this.failed = failed; + this.actionRunConf = actionRunConf; } /** * Check if all given actions are eligible to rerun. * - * @param actions list of CoordinatorActionBean + * @param coordActions list of CoordinatorActionBean * @return true if all actions are eligible to rerun */ private static boolean checkAllActionsRunnable(List<CoordinatorActionBean> coordActions) { @@ -135,11 +140,9 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio * Cleanup output-events directories * * @param eAction coordinator action xml - * @param user user name - * @param group group name */ @SuppressWarnings("unchecked") - private void cleanupOutputEvents(Element eAction, String user, String group, CoordinatorAction action) + private void cleanupOutputEvents(Element eAction) throws CommandException { Element outputList = eAction.getChild("output-events", eAction.getNamespace()); if (outputList != null) { @@ -163,7 +166,7 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio URIHandler handler = Services.get().get(URIHandlerService.class).getURIHandler(uri); String schemeWithAuthority = uri.getScheme() + "://" + uri.getAuthority(); if (!contextMap.containsKey(schemeWithAuthority)) { - Context context = handler.getContext(uri, actionConf, user, false); + Context context = handler.getContext(uri, actionConf, coordJob.getUser(), false); contextMap.put(schemeWithAuthority, context); } handler.delete(uri, contextMap.get(schemeWithAuthority)); @@ -243,6 +246,18 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio coordAction.setLastModifiedTime(new Date()); coordAction.setErrorCode(""); coordAction.setErrorMessage(""); + + // Pushing the configuration which passed through rerun. + if(actionRunConf != null && actionRunConf.size() > 0) { + Configuration createdConf = null; + if(coordAction.getCreatedConf() != null ) { + createdConf = new XConfiguration(new StringReader(coordAction.getCreatedConf())); + } else { + createdConf = new Configuration(); + } + createdConf.set(RERUN_CONF, XmlUtils.prettyPrint(actionRunConf).toString()); + coordAction.setCreatedConf(XmlUtils.prettyPrint(createdConf).toString()); + } updateList.add(new UpdateEntry<CoordActionQuery>(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, coordAction)); writeActionRegistration(coordAction.getActionXml(), coordAction, coordJob.getUser(), coordJob.getGroup()); } @@ -351,7 +366,7 @@ public class CoordRerunXCommand extends RerunTransitionXCommand<CoordinatorActio String actionXml = coordAction.getActionXml(); if (!noCleanup) { Element eAction = XmlUtils.parseXml(actionXml); - cleanupOutputEvents(eAction, coordJob.getUser(), coordJob.getGroup(), coordAction); + cleanupOutputEvents(eAction); } if (refresh) { refreshAction(coordJob, coordAction); http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java index 6506028..eed7ca1 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java @@ -675,7 +675,7 @@ public class V1JobServlet extends BaseJobServlet { throw new CommandException(ErrorCode.E1018, "date or action expected."); } CoordinatorActionInfo coordInfo = coordEngine.reRun(jobId, rerunType, scope, Boolean.valueOf(refresh), - Boolean.valueOf(noCleanup), Boolean.valueOf(failed)); + Boolean.valueOf(noCleanup), Boolean.valueOf(failed), conf); List<CoordinatorActionBean> coordActions; if (coordInfo != null) { coordActions = coordInfo.getCoordActions(); http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java index ae7cdba..45457d1 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java @@ -24,6 +24,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; +import java.io.StringReader; import java.net.URI; import java.util.Date; import java.util.List; @@ -54,7 +55,6 @@ import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor; import org.apache.oozie.executor.jpa.CoordJobInsertJPAExecutor; import org.apache.oozie.executor.jpa.CoordJobQueryExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; -import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.SchemaService; import org.apache.oozie.service.Services; @@ -68,6 +68,7 @@ import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XLog; import org.apache.oozie.util.XmlUtils; +import org.apache.oozie.util.XConfiguration; import org.jdom.Element; import org.jdom.JDOMException; @@ -764,8 +765,8 @@ public class TestCoordRerunXCommand extends XDataTestCase { assertEquals(Job.Status.FAILED, job.getStatus()); try { - new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false) - .call(); + new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, + false, null).call(); fail("Coordinator job is FAILED, rerun should throw exception"); } catch (CommandException ce) { @@ -792,8 +793,8 @@ public class TestCoordRerunXCommand extends XDataTestCase { job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.DONEWITHERROR, job.getStatus()); - new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false) - .call(); + new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false, + null).call(); job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.RUNNINGWITHERROR, job.getStatus()); @@ -815,8 +816,8 @@ public class TestCoordRerunXCommand extends XDataTestCase { job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.PAUSED, job.getStatus()); - new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false) - .call(); + new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false, + null).call(); job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.PAUSED, job.getStatus()); @@ -840,8 +841,8 @@ public class TestCoordRerunXCommand extends XDataTestCase { job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.PAUSEDWITHERROR, job.getStatus()); - new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false) - .call(); + new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, true, false, + null).call(); job = jpaService.execute(coordJobGetExecutor); assertEquals(Job.Status.PAUSEDWITHERROR, job.getStatus()); @@ -1489,16 +1490,18 @@ public class TestCoordRerunXCommand extends XDataTestCase { String externalId = coordClient.getCoordActionInfo(actionId).getExternalId(); - coordClient.reRunCoord(coordJob.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, true); + coordClient.reRunCoord(coordJob.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, true, + new Properties()); - waitFor(150*1000, new Predicate() { + waitFor(150 * 1000, new Predicate() { public boolean evaluate() throws Exception { return (coordClient.getCoordActionInfo(actionId).getStatus() == CoordinatorAction.Status.SUCCEEDED); } }); assertEquals(externalId,coordClient.getCoordActionInfo(actionId).getExternalId()); - coordClient.reRunCoord(coordJob.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false); + coordClient.reRunCoord(coordJob.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false, + new Properties()); waitFor(150*1000, new Predicate() { public boolean evaluate() throws Exception { @@ -1507,4 +1510,61 @@ public class TestCoordRerunXCommand extends XDataTestCase { }); assertNotSame(externalId,coordClient.getCoordActionInfo(actionId).getExternalId()); } + + /** + * Passing config of workflow during rerun of coordinator. + * @throws Exception + */ + public void testCoordRerunWithConfOption() throws Exception { + Date start = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z"); + Date end = DateUtils.parseDateOozieTZ("2009-02-01T23:59Z"); + CoordinatorJobBean coordJob = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, start, end, false, + false, 1); + + CoordinatorActionBean action = addRecordToWithLazyAction(coordJob.getId(), 1, + CoordinatorAction.Status.SUBMITTED, "coord-rerun-action1.xml"); + final String actionId = action.getId(); + new CoordActionStartXCommand(actionId, getTestUser(), "myapp", "myjob").call(); + + final JPAService jpaService = Services.get().get(JPAService.class); + action = jpaService.execute(new CoordActionGetJPAExecutor(actionId)); + + if (action.getStatus() == CoordinatorAction.Status.SUBMITTED) { + fail("CoordActionStartCommand didn't work because the status for action id" + actionId + " is :" + + action.getStatus() + " expected to be NOT SUBMITTED (i.e. RUNNING)"); + } + + final OozieClient coordClient = LocalOozie.getCoordClient(); + final OozieClient wclient = LocalOozie.getClient(); + waitFor(15*1000, new Predicate() { + public boolean evaluate() throws Exception { + return (coordClient.getCoordActionInfo(actionId).getStatus() == CoordinatorAction.Status.RUNNING); + } + }); + + wclient.kill(coordClient.getCoordActionInfo(actionId).getExternalId()); + + waitFor(150*1000, new Predicate() { + public boolean evaluate() throws Exception { + return (coordClient.getCoordActionInfo(actionId).getStatus() == CoordinatorAction.Status.KILLED); + } + }); + + Properties prop = new Properties(); + + // Passing props to coordinator which will be passed to workflow rerun as well. + prop.setProperty("workflowConf", "foo"); + coordClient.reRunCoord(coordJob.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, true, + prop); + + waitFor(150 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return (coordClient.getCoordActionInfo(actionId).getStatus() == CoordinatorAction.Status.SUCCEEDED); + } + }); + + WorkflowJob wfJob = wclient.getJobInfo(coordClient.getCoordActionInfo(actionId).getExternalId()); + Configuration conf = new XConfiguration(new StringReader(wfJob.getConf())); + assertEquals(prop.get("workflowConf"), conf.get("workflowConf")); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java index bdfaf7f..14f5294 100644 --- a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java +++ b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java @@ -337,8 +337,8 @@ public class TestEventGeneration extends XDataTestCase { action.setStatus(CoordinatorAction.Status.KILLED); CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action); queue.clear(); - new CoordRerunXCommand(coord.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false) - .call(); + new CoordRerunXCommand(coord.getId(), RestConstants.JOB_COORD_SCOPE_ACTION, "1", false, true, false, + null).call(); waitFor(3 * 100, new Predicate() { @Override public boolean evaluate() throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java index 01097d3..ca6ae19 100644 --- a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java +++ b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java @@ -177,7 +177,7 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService { } @Override public CoordinatorActionInfo reRun(String jobId, String rerunType, String scope, boolean refresh, - boolean noCleanup, boolean failed) throws BaseEngineException { + boolean noCleanup, boolean failed, Configuration conf) throws BaseEngineException { did = RestConstants.JOB_COORD_ACTION_RERUN; int idx = validateCoordinatorIdx(jobId); started.set(idx, true); http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/core/src/test/java/org/apache/oozie/sla/TestSLAEventGeneration.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLAEventGeneration.java b/core/src/test/java/org/apache/oozie/sla/TestSLAEventGeneration.java index 10a8311..ea82baa 100644 --- a/core/src/test/java/org/apache/oozie/sla/TestSLAEventGeneration.java +++ b/core/src/test/java/org/apache/oozie/sla/TestSLAEventGeneration.java @@ -333,7 +333,7 @@ public class TestSLAEventGeneration extends XDataTestCase { try { new CoordRerunXCommand(job.getId(), RestConstants.JOB_COORD_SCOPE_DATE, "2009-12-15T01:00Z", false, - true, false).call(); + true, false, null).call(); } catch (CommandException ce) { if (ce.getErrorCode() == ErrorCode.E0604) { http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki index 15568a0..fe23156 100644 --- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki +++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki @@ -3489,7 +3489,7 @@ If you add *sla* tags to the Coordinator or Workflow XML files, then the SLA inf * TBD ---++ 13. Web Services API - +` See the [[WebServicesAPI][Web Services API]] page. ---++ 14. Coordinator Rerun @@ -3499,6 +3499,7 @@ Example: <verbatim> $oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] +[-config <arg> (job configuration file '.xml' or '.properties', this file can used to supply properties, which can be used for workflow)] [-action 1, 3-4, 7-40] (-action or -date is required to rerun.) [-date 2009-01-01T01:00Z::2009-05-31T23:59Z, 2009-11-10T01:00Z, 2009-12-31T22:00Z] (if neither -action nor -date is given, the exception will be thrown.) http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 5d4132f..4ac73f9 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -451,7 +451,7 @@ Refer to the [[DG_WorkflowReRun][Rerunning Workflow Jobs]] for details on rerun. Example: <verbatim> -$oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] +$oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] [-config <arg>] [-action 1, 3-4, 7-40] (-action or -date is required to rerun.) [-date 2009-01-01T01:00Z::2009-05-31T23:59Z, 2009-11-10T01:00Z, 2009-12-31T22:00Z] (if neither -action nor -date is given, the exception will be thrown.) http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/docs/src/site/twiki/DG_CoordinatorRerun.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CoordinatorRerun.twiki b/docs/src/site/twiki/DG_CoordinatorRerun.twiki index 12a757c..c937da5 100644 --- a/docs/src/site/twiki/DG_CoordinatorRerun.twiki +++ b/docs/src/site/twiki/DG_CoordinatorRerun.twiki @@ -19,7 +19,7 @@ ---++ Rerun Arguments <verbatim> -$oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] +$oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] [-config <arg>] [-action 1, 3-4, 7-40] (-action or -date is required to rerun.) [-date 2009-01-01T01:00Z::2009-05-31T23:59Z, 2009-11-10T01:00Z, 2009-12-31T22:00Z] (if neither -action nor -date is given, the exception will be thrown.) @@ -35,6 +35,7 @@ $oozie job -rerun <coord_Job_id> [-nocleanup] [-refresh] [-failed] * If -refresh is set, new dataset is re-evaluated for latest() and future(). * If -refresh is set, all dependencies will be re-checked; otherwise only missed dependencies will be checked. * If -failed is set, re-runs the failed workflow actions of the coordinator actions. + * -config can be used to supply properties to workflow by job configuration file '.xml' or '.properties'. ---++ Rerun coordinator actions http://git-wip-us.apache.org/repos/asf/oozie/blob/d1c475c3/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index dac246d..26a02bd 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2092 Provide option to supply config to workflow during rerun of coordinator (jaydeepvishwakarma via shwethags) OOZIE-2100 Publish oozie-webapp artifact (sureshms via bzhang) OOZIE-1889 Convert NamedNativeQueries to JPQL (dvillegas via shwethags) OOZIE-1876 use pom properties rather than specific version numbers in the pom files of hbaselibs, hcataloglibs, sharelib, etc (shwethags)
