Repository: oozie Updated Branches: refs/heads/master fc9aeb13a -> f83f484a1
OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f83f484a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f83f484a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f83f484a Branch: refs/heads/master Commit: f83f484a1a34c9446cf43ba7d627aefabaf58921 Parents: fc9aeb1 Author: Rohini Palaniswamy <[email protected]> Authored: Tue Jun 7 08:32:13 2016 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue Jun 7 08:32:13 2016 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 12 ++++--- .../org/apache/oozie/workflow/lite/NodeDef.java | 5 ++- .../action/hadoop/ActionExecutorTestCase.java | 2 -- .../action/hadoop/TestJavaActionExecutor.java | 38 +++++++++++++++++++- .../command/wf/TestActionCheckXCommand.java | 1 - .../command/wf/TestActionStartXCommand.java | 4 --- .../apache/oozie/event/TestEventGeneration.java | 1 - .../oozie/service/TestRecoveryService.java | 1 - .../apache/oozie/store/TestDBWorkflowStore.java | 1 - .../org/apache/oozie/test/XDataTestCase.java | 1 - release-log.txt | 1 + 11 files changed, 49 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 6893bb8..639003e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1307,11 +1307,13 @@ public class JavaActionExecutor extends ActionExecutor { HashMap<String, CredentialsProperties> props = new HashMap<String, CredentialsProperties>(); if (context != null && action != null) { String credsInAction = action.getCred(); - LOG.debug("Get credential '" + credsInAction + "' properties for action : " + action.getId()); - String[] credNames = credsInAction.split(","); - for (String credName : credNames) { - CredentialsProperties credProps = getCredProperties(context, credName); - props.put(credName, credProps); + if (credsInAction != null) { + LOG.debug("Get credential '" + credsInAction + "' properties for action : " + action.getId()); + String[] credNames = credsInAction.split(","); + for (String credName : credNames) { + CredentialsProperties credProps = getCredProperties(context, credName); + props.put(credName, credProps); + } } } else { http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java index a395b77..9e66d28 100644 --- a/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java +++ b/core/src/main/java/org/apache/oozie/workflow/lite/NodeDef.java @@ -40,7 +40,7 @@ public class NodeDef implements Writable { private Class<? extends NodeHandler> handlerClass; private String conf = null; private List<String> transitions = new ArrayList<String>(); - private String cred = "null"; + private String cred = null; private String userRetryMax = "null"; private String userRetryInterval = "null"; @@ -154,6 +154,9 @@ public class NodeDef implements Writable { nodeDefVersion = LiteWorkflowStoreService.NODE_DEF_VERSION_1; name = dataInput.readUTF(); cred = dataInput.readUTF(); + if (cred.equals("null")) { + cred = null; + } String handlerClassName = dataInput.readUTF(); if ((handlerClassName != null) && (handlerClassName.length() > 0)) { try { http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java index e1c450c..d74160a 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java @@ -262,7 +262,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase { WorkflowActionBean action = new WorkflowActionBean(); action.setName(actionName); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName)); workflow.getActions().add(action); return workflow; @@ -296,7 +295,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase { WorkflowActionBean action = new WorkflowActionBean(); action.setName(actionName); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName)); workflow.getActions().add(action); return workflow; http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 057c9fb..85bb993 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FSDataOutputStream; @@ -1025,6 +1026,42 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } } + + public void testCredentialsWithoutCredTag() throws Exception { + // create a workflow with credentials + // add a pig action without cred tag + String workflowXml = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='pig-wf'>" + "<credentials>" + + "<credential name='abcname' type='abc'>" + "<property>" + "<name>property1</name>" + + "<value>value1</value>" + "</property>" + "<property>" + "<name>property2</name>" + + "<value>value2</value>" + "</property>" + "<property>" + "<name>${property3}</name>" + + "<value>${value3}</value>" + "</property>" + "</credential>" + "</credentials>" + + "<start to='pig1' />" + "<action name='pig1'>" + "<pig>" + "</pig>" + + "<ok to='end' />" + "<error to='fail' />" + "</action>" + "<kill name='fail'>" + + "<message>Pig failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>" + + "<end name='end' />" + "</workflow-app>"; + + JavaActionExecutor ae = new JavaActionExecutor(); + WorkflowJobBean wfBean = addRecordToWfJobTable("test1", workflowXml); + WorkflowActionBean action = (WorkflowActionBean) wfBean.getActions().get(0); + action.setType(ae.getType()); + String actionXml = "<pig>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>" + + "<prepare>" + "<delete path='outputdir' />" + "</prepare>" + "<configuration>" + "<property>" + + "<name>mapred.compress.map.output</name>" + "<value>true</value>" + "</property>" + "<property>" + + "<name>mapred.job.queue.name</name>" + "<value>${queueName}</value>" + "</property>" + + "</configuration>" + "<script>org/apache/oozie/examples/pig/id.pig</script>" + + "<param>INPUT=${inputDir}</param>" + "<param>OUTPUT=${outputDir}/pig-output</param>" + "</pig>"; + action.setConf(actionXml); + Context context = new Context(wfBean, action); + + Element actionXmlconf = XmlUtils.parseXml(action.getConf()); + // action job configuration + Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf); + + // should not throw JA021 exception + HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, action, + actionConf); + } + public void testCredentialsSkip() throws Exception { // Try setting oozie.credentials.skip at different levels, and verifying the correct behavior // oozie-site: false -- job-level: null -- action-level: null @@ -1163,7 +1200,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { wfBean.setStatus(WorkflowJob.Status.SUCCEEDED); WorkflowActionBean action = new WorkflowActionBean(); action.setName("test"); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfBean.getId(), "test")); wfBean.getActions().add(action); return wfBean; http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java index f503b1f..5898d1a 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java @@ -633,7 +633,6 @@ public class TestActionCheckXCommand extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java index b7489e9..ea90c08 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java @@ -334,7 +334,6 @@ public class TestActionStartXCommand extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); @@ -421,7 +420,6 @@ public class TestActionStartXCommand extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); @@ -529,7 +527,6 @@ public class TestActionStartXCommand extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); @@ -568,7 +565,6 @@ public class TestActionStartXCommand extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("fs"); http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java index f662d8a..afd3f8f 100644 --- a/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java +++ b/core/src/test/java/org/apache/oozie/event/TestEventGeneration.java @@ -686,7 +686,6 @@ public class TestEventGeneration extends XDataTestCase { action.setStartTime(new Date()); action.setEndTime(new Date()); action.setLastCheckTime(new Date()); - action.setCred("null"); action.setPendingOnly(); String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java index 13d8e8d..eab177b 100644 --- a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java +++ b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java @@ -864,7 +864,6 @@ public class TestRecoveryService extends XDataTestCase { WorkflowActionBean action = new WorkflowActionBean(); String actionname = "testAction"; action.setName(actionname); - action.setCred("null"); action.setId(Services.get().get(UUIDService.class).generateChildId(wfId, actionname)); action.setJobId(wfId); action.setType("map-reduce"); http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java b/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java index c263e86..b571fb5 100644 --- a/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java +++ b/core/src/test/java/org/apache/oozie/store/TestDBWorkflowStore.java @@ -477,7 +477,6 @@ public class TestDBWorkflowStore extends XTestCase { a31.setId(str.toString()); a31.setJobId(wfBean3.getId()); a31.setStatus(WorkflowAction.Status.PREP); - a31.setCred("null"); store.beginTrx(); store.insertAction(a31); store.commitTrx(); http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java index a9aa79a..081d2f1 100644 --- a/core/src/test/java/org/apache/oozie/test/XDataTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XDataTestCase.java @@ -1417,7 +1417,6 @@ public abstract class XDataTestCase extends XHCatTestCase { action.setStartTime(currDate); action.setEndTime(currDate); action.setLastCheckTime(currDate); - action.setCred("null"); action.setStats("dummyStats"); if (pending) { action.setPending(); http://git-wip-us.apache.org/repos/asf/oozie/blob/f83f484a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 189ca21..feea868 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2553 Cred tag is required for all actions in the workflow even if an action does not require it (me.venkatr via rohini) OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru) OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru) OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru)
