FALCON-1720 Rerun API does not rerun succeeded instances (by Pavan Kolamuri)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ccb6df38 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ccb6df38 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ccb6df38 Branch: refs/heads/master Commit: ccb6df38bf086135d28bce9c0c92d6fd23e19459 Parents: 7cde36c Author: Pallavi Rao <[email protected]> Authored: Wed Jan 6 15:35:31 2016 +0530 Committer: Pallavi Rao <[email protected]> Committed: Wed Jan 6 15:35:31 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../falcon/workflow/engine/OozieDAGEngine.java | 24 +++++++++++++++----- .../AbstractSchedulerManagerJerseyIT.java | 2 +- .../InstanceSchedulerManagerJerseyIT.java | 12 +++++++++- .../local-process-noinputs-template.xml | 2 +- 5 files changed, 33 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2ed1ab4..5677175 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -90,6 +90,8 @@ Proposed Release Version: 0.9 OPTIMIZATIONS BUG FIXES + FALCON-1720 Rerun API does not rerun succeeded instances (Pavan Kolamuri via Pallavi Rao) + FALCON-1719 Retry does not update the state of the instance in the database (Pavan Kolamuri via Pallavi Rao) FALCON-1710 dependency API sets totalResults as 0 by default(Praveen Adlakha via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java ---------------------------------------------------------------------- diff --git a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java index 1425a97..1d0e126 100644 --- a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java +++ b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java @@ -237,17 +237,29 @@ public class OozieDAGEngine implements DAGEngine { String jobId = instance.getExternalID(); try { WorkflowJob jobInfo = client.getJobInfo(jobId); - Properties jobprops = OozieUtils.toProperties(jobInfo.getConf()); - if (props != null) { - jobprops.putAll(props); + if (props == null) { + props = new Properties(); } //if user has set any of these oozie rerun properties then force rerun flag is ignored - if (!jobprops.containsKey(OozieClient.RERUN_FAIL_NODES) - && !jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) { - jobprops.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced)); + if (!props.containsKey(OozieClient.RERUN_FAIL_NODES) + && !props.containsKey(OozieClient.RERUN_SKIP_NODES)) { + props.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced)); } + + Properties jobprops = OozieUtils.toProperties(jobInfo.getConf()); + jobprops.putAll(props); + jobprops.remove(OozieClient.COORDINATOR_APP_PATH); jobprops.remove(OozieClient.BUNDLE_APP_PATH); + // In case if both props exists one should be removed otherwise it will fail. + // This case will occur when user runs workflow with skip-nodes property and + // try to do force rerun or rerun with fail-nodes property. + if (jobprops.containsKey(OozieClient.RERUN_FAIL_NODES) + && jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) { + LOG.warn("Both " + OozieClient.RERUN_SKIP_NODES + " and " + OozieClient.RERUN_FAIL_NODES + + " are present in workflow params removing" + OozieClient.RERUN_SKIP_NODES); + jobprops.remove(OozieClient.RERUN_SKIP_NODES); + } client.reRun(jobId, jobprops); assertStatus(instance.getExternalID(), Job.Status.PREP, Job.Status.RUNNING, Job.Status.SUCCEEDED); LOG.info("Rerun job {} of entity {} of time {} on cluster {}", jobId, instance.getEntity().getName(), http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java index f053b76..0a3e984 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java @@ -62,7 +62,7 @@ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase { private static final String DB_BASE_DIR = "target/test-data/falcondb"; protected static String dbLocation = DB_BASE_DIR + File.separator + "data.db"; protected static String url = "jdbc:derby:"+ dbLocation +";create=true"; - protected static final String DB_SQL_FILE = DB_BASE_DIR + File.separator + "out.sql"; + protected static final String DB_SQL_FILE = dbLocation + File.separator + "out.sql"; protected LocalFileSystem localFS = new LocalFileSystem(); http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java index 1523b76..18c36ff 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java +++ b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java @@ -64,7 +64,7 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe } @Test - public void testKillInstances() throws Exception { + public void testKillAndRerunInstances() throws Exception { UnitTestContext context = new UnitTestContext(); Map<String, String> overlay = context.getUniqueOverlay(); @@ -84,6 +84,16 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe processName, START_INSTANCE); Assert.assertEquals(status, InstancesResult.WorkflowStatus.KILLED); + result = falconUnitClient.rerunInstances(EntityType.PROCESS.toString(), + processName, START_INSTANCE, END_TIME, colo, null, null, null, null, true, null); + assertStatus(result); + + waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE, + InstancesResult.WorkflowStatus.RUNNING); + status = getClient().getInstanceStatus(EntityType.PROCESS.name(), + processName, START_INSTANCE); + Assert.assertEquals(status, InstancesResult.WorkflowStatus.RUNNING); + } http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/resources/local-process-noinputs-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/local-process-noinputs-template.xml b/webapp/src/test/resources/local-process-noinputs-template.xml index aabdc6a..a7388d8 100644 --- a/webapp/src/test/resources/local-process-noinputs-template.xml +++ b/webapp/src/test/resources/local-process-noinputs-template.xml @@ -38,5 +38,5 @@ <property name="sundayThisWeek" value="${currentWeek('SUN', 0, 0)}"/> </properties> <workflow engine="oozie" path="##workflow.path##" lib="##workflow.lib.path##"/> - <retry policy="periodic" delay="minutes(10)" attempts="3"/> + <retry policy="periodic" delay="minutes(10)" attempts="1"/> </process>
