Repository: oozie Updated Branches: refs/heads/master ee8710bde -> 0cf2fb3e7
OOZIE-2439 FS Action no longer uses name-node from global section or default NN (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cf2fb3e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cf2fb3e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cf2fb3e Branch: refs/heads/master Commit: 0cf2fb3e78fcc46d30270fcd86248b023a90c871 Parents: ee8710b Author: Robert Kanter <[email protected]> Authored: Fri Jan 15 10:39:43 2016 -0800 Committer: Robert Kanter <[email protected]> Committed: Fri Jan 15 10:39:43 2016 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/FsActionExecutor.java | 7 +- .../workflow/lite/LiteWorkflowAppParser.java | 14 +++- .../lite/TestLiteWorkflowAppParser.java | 85 ++++++++++++++++++++ .../wf-schema-fs-no-namenode-global.xml | 47 +++++++++++ .../test/resources/wf-schema-fs-no-namenode.xml | 44 ++++++++++ release-log.txt | 1 + 6 files changed, 196 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java index b8649ac..121cd49 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java @@ -58,10 +58,12 @@ import org.jdom.Element; */ public class FsActionExecutor extends ActionExecutor { + public static final String ACTION_TYPE = "fs"; + private final int maxGlobCount; public FsActionExecutor() { - super("fs"); + super(ACTION_TYPE); maxGlobCount = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX); } @@ -645,4 +647,7 @@ public class FsActionExecutor extends ActionExecutor { } } + public boolean supportsConfigurationJobXML() { + return true; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java index 03c84f1..7f1c73c 100644 --- a/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java +++ b/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java @@ -20,6 +20,7 @@ package org.apache.oozie.workflow.lite; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.io.Writable; +import org.apache.oozie.action.hadoop.FsActionExecutor; import org.apache.oozie.action.oozie.SubWorkflowActionExecutor; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.util.ELUtils; @@ -737,7 +738,13 @@ public class LiteWorkflowAppParser { Namespace actionNs = actionElement.getNamespace(); + // If this is the global section or ActionExecutor.requiresNameNodeJobTracker() returns true, we parse the action's + // <name-node> and <job-tracker> fields. If those aren't defined, we take them from the <global> section. If those + // aren't defined, we take them from the oozie-site defaults. If those aren't defined, we throw a WorkflowException. + // However, for the SubWorkflow and FS Actions, as well as the <global> section, we don't throw the WorkflowException. + // Also, we only parse the NN (not the JT) for the FS Action. if (SubWorkflowActionExecutor.ACTION_TYPE.equals(actionElement.getName()) || + FsActionExecutor.ACTION_TYPE.equals(actionElement.getName()) || GLOBAL.equals(actionElement.getName()) || ae.requiresNameNodeJobTracker()) { if (actionElement.getChild(NAME_NODE, actionNs) == null) { if (gData != null && gData.nameNode != null) { @@ -745,11 +752,13 @@ public class LiteWorkflowAppParser { } else if (defaultNameNode != null) { addChildElement(actionElement, actionNs, NAME_NODE, defaultNameNode); } else if (!(SubWorkflowActionExecutor.ACTION_TYPE.equals(actionElement.getName()) || + FsActionExecutor.ACTION_TYPE.equals(actionElement.getName()) || GLOBAL.equals(actionElement.getName()))) { throw new WorkflowException(ErrorCode.E0701, "No " + NAME_NODE + " defined"); } } - if (actionElement.getChild(JOB_TRACKER, actionNs) == null) { + if (actionElement.getChild(JOB_TRACKER, actionNs) == null && + !FsActionExecutor.ACTION_TYPE.equals(actionElement.getName())) { if (gData != null && gData.jobTracker != null) { addChildElement(actionElement, actionNs, JOB_TRACKER, gData.jobTracker); } else if (defaultJobTracker != null) { @@ -761,6 +770,9 @@ public class LiteWorkflowAppParser { } } + // If this is the global section or ActionExecutor.supportsConfigurationJobXML() returns true, we parse the action's + // <configuration> and <job-xml> fields. We also merge this with those from the <global> section, if given. If none are + // defined, empty values are placed. Exceptions are thrown if there's an error parsing, but not if they're not given. if ( GLOBAL.equals(actionElement.getName()) || ae.supportsConfigurationJobXML()) { @SuppressWarnings("unchecked") List<Element> actionJobXmls = actionElement.getChildren(JOB_XML, actionNs); http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index 1fc1736..1983a8c 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -510,6 +510,91 @@ public class TestLiteWorkflowAppParser extends XTestCase { } } + public void testParserFsGlobalNN() throws Exception { + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteControlNodeHandler.class, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode-global.xml", -1), + new Configuration()); + String a = app.getNode("a").getConf(); + String expectedA = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <name-node>action-nn</name-node>\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <configuration />\r\n" + + "</fs>"; + a = cleanupXml(a); + assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", "")); + String b = app.getNode("b").getConf(); + String expectedB = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <name-node>global-nn</name-node>\r\n" + + " <configuration />\r\n" + + "</fs>"; + b = cleanupXml(b); + assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", "")); + } + + public void testParserFsDefaultNN() throws Exception { + ConfigurationService.set("oozie.actions.default.name-node", "default-nn"); + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteControlNodeHandler.class, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode.xml", -1), + new Configuration()); + String a = app.getNode("a").getConf(); + String expectedA = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <name-node>action-nn</name-node>\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <configuration />\r\n" + + "</fs>"; + a = cleanupXml(a); + assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", "")); + String b = app.getNode("b").getConf(); + String expectedB = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <name-node>default-nn</name-node>\r\n" + + " <configuration />\r\n" + + "</fs>"; + b = cleanupXml(b); + assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", "")); + } + + public void testParserFsNoNN() throws Exception { + LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, + LiteWorkflowStoreService.LiteControlNodeHandler.class, + LiteWorkflowStoreService.LiteDecisionHandler.class, + LiteWorkflowStoreService.LiteActionHandler.class); + + LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-fs-no-namenode.xml", -1), + new Configuration()); + String a = app.getNode("a").getConf(); + String expectedA = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <name-node>action-nn</name-node>\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <configuration />\r\n" + + "</fs>"; + a = cleanupXml(a); + assertEquals(expectedA.replaceAll(" ", ""), a.replaceAll(" ", "")); + // The FS Action shouldn't care if there's no NN in the end + String b = app.getNode("b").getConf(); + String expectedB = + "<fs xmlns=\"uri:oozie:workflow:0.4\">\r\n" + + " <mkdir path=\"/foo\" />\r\n" + + " <configuration />\r\n" + + "</fs>"; + b = cleanupXml(b); + assertEquals(expectedB.replaceAll(" ", ""), b.replaceAll(" ", "")); + } + public void testParser() throws Exception { LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, LiteWorkflowStoreService.LiteControlNodeHandler.class, http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/resources/wf-schema-fs-no-namenode-global.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-schema-fs-no-namenode-global.xml b/core/src/test/resources/wf-schema-fs-no-namenode-global.xml new file mode 100644 index 0000000..d5afb58 --- /dev/null +++ b/core/src/test/resources/wf-schema-fs-no-namenode-global.xml @@ -0,0 +1,47 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.4" name="test-wf"> + <global> + <name-node>global-nn</name-node> + </global> + + <start to="a"/> + + <action name="a"> + <fs> + <name-node>action-nn</name-node> + <mkdir path="/foo"/> + </fs> + <ok to="b"/> + <error to="c"/> + </action> + + <action name="b"> + <fs> + <mkdir path="/foo"/> + </fs> + <ok to="d"/> + <error to="c"/> + </action> + + <kill name="c"> + <message>fail</message> + </kill> + + <end name="d"/> +</workflow-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/core/src/test/resources/wf-schema-fs-no-namenode.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-schema-fs-no-namenode.xml b/core/src/test/resources/wf-schema-fs-no-namenode.xml new file mode 100644 index 0000000..5b42b7c --- /dev/null +++ b/core/src/test/resources/wf-schema-fs-no-namenode.xml @@ -0,0 +1,44 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.4" name="test-wf"> + + <start to="a"/> + + <action name="a"> + <fs> + <name-node>action-nn</name-node> + <mkdir path="/foo"/> + </fs> + <ok to="b"/> + <error to="c"/> + </action> + + <action name="b"> + <fs> + <mkdir path="/foo"/> + </fs> + <ok to="d"/> + <error to="c"/> + </action> + + <kill name="c"> + <message>fail</message> + </kill> + + <end name="d"/> +</workflow-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/0cf2fb3e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 1355d65..12a0641 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2439 FS Action no longer uses name-node from global section or default NN (rkanter) OOZIE-2435 TestCoordChangeXCommand is flakey (fdenes via rkanter) OOZIE-2428 TestSLAService, TestSLAEventGeneration flakey tests (fdenes via rkanter) OOZIE-2437 Exclude additional Hadoop dependencies from Spark Sharelib (rkanter)
