This is an automated email from the ASF dual-hosted git repository.
asalamon74 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push:
new b6d3dd6 OOZIE-3265 Properties RERUN_FAIL_NODES and RERUN_SKIP_NODES
should be able to appear together (kmarton via asalamon74)
b6d3dd6 is described below
commit b6d3dd662d0777ac7ebb72e609acfdc0bb9b4b83
Author: Andras Salamon <[email protected]>
AuthorDate: Fri Jul 19 11:24:22 2019 +0200
OOZIE-3265 Properties RERUN_FAIL_NODES and RERUN_SKIP_NODES should be able
to appear together (kmarton via asalamon74)
---
core/src/main/java/org/apache/oozie/DagEngine.java | 15 +--
.../test/java/org/apache/oozie/TestDagEngine.java | 131 +++++++++++++--------
.../apache/oozie/command/wf/TestReRunXCommand.java | 6 -
docs/src/site/markdown/DG_WorkflowReRun.md | 14 ++-
release-log.txt | 1 +
5 files changed, 96 insertions(+), 71 deletions(-)
diff --git a/core/src/main/java/org/apache/oozie/DagEngine.java
b/core/src/main/java/org/apache/oozie/DagEngine.java
index 70ce961..7de8385 100644
--- a/core/src/main/java/org/apache/oozie/DagEngine.java
+++ b/core/src/main/java/org/apache/oozie/DagEngine.java
@@ -30,7 +30,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.OozieClient;
@@ -60,6 +60,7 @@ import org.apache.oozie.executor.jpa.JPAExecutorException;
import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
import org.apache.oozie.service.CallableQueueService;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.DagXLogInfoService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.XLogService;
@@ -297,15 +298,15 @@ public class DagEngine extends BaseEngine {
}
}
- private void validateReRunConfiguration(Configuration conf) throws
DagEngineException {
+ @VisibleForTesting
+ protected void validateReRunConfiguration(Configuration conf) throws
DagEngineException {
if (conf.get(OozieClient.APP_PATH) == null) {
throw new DagEngineException(ErrorCode.E0401,
OozieClient.APP_PATH);
}
- if (conf.get(OozieClient.RERUN_SKIP_NODES) == null &&
conf.get(OozieClient.RERUN_FAIL_NODES) == null) {
- throw new DagEngineException(ErrorCode.E0401,
OozieClient.RERUN_SKIP_NODES + " OR "
- + OozieClient.RERUN_FAIL_NODES);
- }
- if (conf.get(OozieClient.RERUN_SKIP_NODES) != null &&
conf.get(OozieClient.RERUN_FAIL_NODES) != null) {
+ boolean rerunFailNodes = ConfigurationService.getBoolean(conf,
OozieClient.RERUN_FAIL_NODES);
+ String skipNodes = conf.get(OozieClient.RERUN_SKIP_NODES);
+
+ if (rerunFailNodes && skipNodes != null) {
throw new DagEngineException(ErrorCode.E0404,
OozieClient.RERUN_SKIP_NODES + " OR "
+ OozieClient.RERUN_FAIL_NODES);
}
diff --git a/core/src/test/java/org/apache/oozie/TestDagEngine.java
b/core/src/test/java/org/apache/oozie/TestDagEngine.java
index 2147405..5f71467 100644
--- a/core/src/test/java/org/apache/oozie/TestDagEngine.java
+++ b/core/src/test/java/org/apache/oozie/TestDagEngine.java
@@ -47,6 +47,7 @@ import java.nio.charset.StandardCharsets;
public class TestDagEngine extends XTestCase {
private EmbeddedServletContainer container;
private Services services;
+ private DagEngine engine;
public static class CallbackServlet extends HttpServlet {
public static volatile String JOB_ID = null;
@@ -79,6 +80,7 @@ public class TestDagEngine extends XTestCase {
services = new Services();
services.init();
services.get(ActionService.class).registerAndInitExecutor(ForTestingActionExecutor.class);
+ engine = new DagEngine(getTestUser());
}
protected void tearDown() throws Exception {
@@ -175,67 +177,92 @@ public class TestDagEngine extends XTestCase {
Writer writer = new OutputStreamWriter(new FileOutputStream(new
File(getTestCaseDir(), "workflow" +
".xml")), StandardCharsets.UTF_8);
IOUtils.copyCharStream(reader, writer);
-
final DagEngine engine = new DagEngine(getTestUser());
Configuration conf = new XConfiguration();
conf.set(OozieClient.APP_PATH, workflowPath);
conf.set(OozieClient.USER_NAME, getTestUser());
-
conf.set(OozieClient.LOG_TOKEN, "t");
conf.set("signal-value", "OK");
conf.set("external-status", "ok");
conf.set("error", "end.error");
+ engine.submitJob(conf, true);
+ engine.submitJob(conf, false);
+ }
- final String jobId1 = engine.submitJob(conf, true);
- String jobId2 = engine.submitJob(conf, false);
-/*
- WorkflowsInfo wfInfo = engine.getJobs("group=" + getTestGroup(), 1, 1);
- List<WorkflowJobBean> workflows = wfInfo.getWorkflows();
- assertEquals(1, workflows.size());
- assertEquals(getTestGroup(), workflows.get(0).getGroup());
- assertEquals(jobId1, workflows.get(0).getId());
-
- wfInfo = engine.getJobs("group=" + getTestGroup(), 1, 5);
- workflows = wfInfo.getWorkflows();
- assertEquals(2, workflows.size());
- assertEquals(getTestGroup(), workflows.get(0).getGroup());
- assertEquals(jobId1, workflows.get(0).getId());
- assertEquals(jobId2, workflows.get(1).getId());
-
- wfInfo = engine.getJobs("user=" + getTestUser(), 1, 1);
- workflows = wfInfo.getWorkflows();
- assertEquals(1, workflows.size());
- assertEquals(getTestUser(), workflows.get(0).getUser());
- assertEquals(jobId1, workflows.get(0).getId());
-
- wfInfo = engine.getJobs("user=" + getTestUser(), 2, 5);
- workflows = wfInfo.getWorkflows();
- assertEquals(1, workflows.size());
- assertEquals(getTestUser(), workflows.get(0).getUser());
- assertEquals(jobId2, workflows.get(0).getId());
-
- waitFor(5000, new Predicate() {
- public boolean evaluate() throws Exception {
- WorkflowJobBean bean =
Services.get().get(WorkflowStoreService.class).create().getWorkflow(jobId1,
false);
- return bean.getWorkflowInstance().getStatus().isEndState();
- }
- });
+ public void testValidateRerunConfigurationMissingApplicationPath() {
+ String appPath = null;
+ boolean rerunFailNodes = false;
+ String rerunSkipNodes = null;
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ try {
+ engine.validateReRunConfiguration(rerunConfig);
+ fail("DagEngineException should have been thrown");
+ }
+ catch (DagEngineException e) {
+ assertEquals("Missing expected error code", ErrorCode.E0401,
e.getErrorCode());
+ assertTrue("Missing expected error message",
e.getMessage().contains(OozieClient.APP_PATH));
+ }
+ }
+
+ public void
testValidateRerunConfigurationFalseRerunFailNodesEmptySkipNodes() throws
DagEngineException {
+ String appPath = getTestCaseDir();
+ boolean rerunFailNodes = false;
+ String rerunSkipNodes = null;
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ engine.validateReRunConfiguration(rerunConfig);
+ }
- wfInfo = engine.getJobs("status=PREP", 1, 5);
- workflows = wfInfo.getWorkflows();
- assertEquals(1, workflows.size());
- assertEquals(jobId2, workflows.get(0).getId());
-
- wfInfo = engine.getJobs("name=test-wf", 1, 5);
- workflows = wfInfo.getWorkflows();
- assertEquals(2, workflows.size());
- assertEquals(jobId1, workflows.get(0).getId());
- assertEquals(jobId2, workflows.get(1).getId());
-
- wfInfo = engine.getJobs("name=test-wf;status=PREP", 1, 5);
- workflows = wfInfo.getWorkflows();
- assertEquals(1, workflows.size());
- assertEquals(jobId2, workflows.get(0).getId());
-*/
+ public void
testValidateRerunConfigurationNullRerunFailNodesNullSkipNodes() throws
DagEngineException {
+ String appPath = getTestCaseDir();
+ Boolean rerunFailNodes = null;
+ String rerunSkipNodes = null;
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ engine.validateReRunConfiguration(rerunConfig);
+ }
+
+ public void
testValidateRerunConfigurationTrueRerunFailNodesEmptySkipNodes() throws
DagEngineException {
+ String appPath = getTestCaseDir();
+ boolean rerunFailNodes = true;
+ String rerunSkipNodes = null;
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ engine.validateReRunConfiguration(rerunConfig);
+ }
+
+ public void
testValidateRerunConfigurationFalseRerunFailNodesNonEmptySkipNodes() throws
DagEngineException {
+ String appPath = getTestCaseDir();
+ boolean rerunFailNodes = false;
+ String rerunSkipNodes = "node1";
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ engine.validateReRunConfiguration(rerunConfig);
+ }
+
+ public void
testValidateRerunConfigurationTrueRerunFailNodesNonEmptySkipNodes() {
+ String appPath = getTestCaseDir();
+ boolean rerunFailNodes = true;
+ String rerunSkipNodes = "node1";
+ Configuration rerunConfig = setupRerunConfig(appPath, rerunFailNodes,
rerunSkipNodes);
+ try {
+ engine.validateReRunConfiguration(rerunConfig);
+ fail("DagEngineException should have been thrown");
+ }
+ catch (DagEngineException e) {
+ assertEquals("Missing expected error code",ErrorCode.E0404,
e.getErrorCode());
+ assertTrue("Missing expected error message",
e.getMessage().contains(OozieClient.RERUN_FAIL_NODES));
+ assertTrue("Missing expected error message",
e.getMessage().contains(OozieClient.RERUN_SKIP_NODES));
+ }
+ }
+
+ private Configuration setupRerunConfig(final String appPath, final Boolean
rerunFailNodes, final String rerunSkipNodes) {
+ Configuration rerunConfig = new Configuration();
+ if (appPath != null) {
+ rerunConfig.set(OozieClient.APP_PATH, appPath);
+ }
+ if (rerunFailNodes != null) {
+ rerunConfig.setBoolean(OozieClient.RERUN_FAIL_NODES,
rerunFailNodes);
+ }
+ if (rerunSkipNodes != null) {
+ rerunConfig.setStrings(OozieClient.RERUN_SKIP_NODES,
rerunSkipNodes);
+ }
+ return rerunConfig;
}
}
diff --git
a/core/src/test/java/org/apache/oozie/command/wf/TestReRunXCommand.java
b/core/src/test/java/org/apache/oozie/command/wf/TestReRunXCommand.java
index d1d75b5..e95d339 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestReRunXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestReRunXCommand.java
@@ -350,12 +350,6 @@ public class TestReRunXCommand extends XDataTestCase {
}
});
assertEquals(WorkflowJob.Status.KILLED,
wfClient.getJobInfo(jobId).getStatus());
- try {
- wfClient.reRun(jobId, newConf);
- }
- catch (OozieClientException e) {
-
assertTrue(e.getCause().getMessage().contains(ErrorCode.E0401.toString()));
- }
newConf = wfClient.createConfiguration();
// Skip a non-executed node
getFileSystem().delete(new Path(path, "p2"), true);
diff --git a/docs/src/site/markdown/DG_WorkflowReRun.md
b/docs/src/site/markdown/DG_WorkflowReRun.md
index c128681..a7d2026 100644
--- a/docs/src/site/markdown/DG_WorkflowReRun.md
+++ b/docs/src/site/markdown/DG_WorkflowReRun.md
@@ -2,17 +2,19 @@
[::Go back to Oozie Documentation Index::](index.html)
-# Workflow ReRrun
+# Workflow ReRun
<!-- MACRO{toc|fromDepth=1|toDepth=4} -->
## Configs
* oozie.wf.application.path
- * Only one of following two configurations is mandatory. Both should not be
defined at the same time
- * oozie.wf.rerun.skip.nodes
- * oozie.wf.rerun.failnodes
+ * Only one of following two configurations is mandatory. Both should not be
defined at the same time.
+ * `oozie.wf.rerun.skip.nodes`
+ * `oozie.wf.rerun.failnodes`
* Skip nodes are comma separated list of action names. They can be any
action nodes including decision node.
- * The valid value of `oozie.wf.rerun.failnodes` is true or false.
+ If this value is empty, or null, and `oozie.wf.rerun.failnodes` is false,
than the whole workflow is rerunned.
+ * The valid value of `oozie.wf.rerun.failnodes` is true or false. By
default its value is false. In case of setting
+ this value to true, all the nodes where the status is OK will be skipped.
* If secured hadoop version is used, the following two properties needs to
be specified as well
* mapreduce.jobtracker.kerberos.principal
* dfs.namenode.kerberos.principal.
@@ -26,7 +28,7 @@ $ oozie job -oozie http://localhost:11000/oozie -rerun
14-20090525161321-oozie-j
* Workflow with id wfId should exist.
* Workflow with id wfId should be in SUCCEEDED/KILLED/FAILED.
- * If specified , nodes in the config oozie.wf.rerun.skip.nodes must be
completed successfully.
+ * If specified , nodes in the config `oozie.wf.rerun.skip.nodes` must be
completed successfully.
## ReRun
diff --git a/release-log.txt b/release-log.txt
index 7f5609c..99382d1 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.2.0 release (trunk - unreleased)
+OOZIE-3265 Properties RERUN_FAIL_NODES and RERUN_SKIP_NODES should be able to
appear together (kmarton via asalamon74)
OOZIE-3024 Email action ignores value of content_type attribute when
attachment attribute is set (matijhs via asalamon74)
OOZIE-3526 Global job-xml not being overwritten by job-xml specified for an
action (mgogineni via rohini)
OOZIE-2755 Oozie HA: ZKJobsConcurrencyService throws runtime exception when
numOozies is zero(asalamon74 via kmarton)
