OOZIE-1618 dryrun should check variable substitution in workflow.xml (bowenzhangusa via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c729cab6 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c729cab6 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c729cab6 Branch: refs/heads/master Commit: c729cab6217f55fe26678551a53a7e3532a73356 Parents: 732e8f2 Author: Robert Kanter <[email protected]> Authored: Thu Feb 13 10:55:09 2014 -0800 Committer: Robert Kanter <[email protected]> Committed: Thu Feb 13 10:55:09 2014 -0800 ---------------------------------------------------------------------- .../apache/oozie/command/wf/SubmitXCommand.java | 40 ++++++++++++++++++++ .../oozie/command/wf/TestSubmitXCommand.java | 9 +++++ .../lite/TestLiteWorkflowAppParser.java | 5 +-- .../test/resources/wf-schema-valid-global.xml | 3 +- release-log.txt | 1 + 5 files changed, 53 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c729cab6/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java index 0a68673..66baca1 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.oozie.AppType; import org.apache.oozie.SLAEventBean; +import org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.ErrorCode; import org.apache.oozie.action.oozie.SubWorkflowActionExecutor; @@ -59,8 +60,11 @@ import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.client.SLAEvent.SlaAppType; import org.apache.oozie.client.rest.JsonBean; import org.jdom.Element; +import org.jdom.filter.ElementFilter; + import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -238,6 +242,29 @@ public class SubmitXCommand extends WorkflowXCommand<String> { return workflow.getId(); } else { + // Checking variable substitution for dryrun + ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false); + Element workflowXml = XmlUtils.parseXml(app.getDefinition()); + removeSlaElements(workflowXml); + String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString()); + workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class); + workflowXml = XmlUtils.parseXml(workflowXmlString); + + Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml")); + + // Checking all variable substitutions in job-xml files + while (it.hasNext()) { + Element e = it.next(); + String jobXml = e.getTextTrim(); + Path xmlPath = new Path(workflow.getAppPath(), jobXml); + Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath)); + + + String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString(); + jobXmlConfString = XmlUtils.removeComments(jobXmlConfString); + context.getELEvaluator().evaluate(jobXmlConfString, String.class); + } + return "OK"; } } @@ -252,6 +279,19 @@ public class SubmitXCommand extends WorkflowXCommand<String> { } } + private void removeSlaElements(Element eWfJob) { + Element sla = XmlUtils.getSLAElement(eWfJob); + if (sla != null) { + eWfJob.removeChildren(sla.getName(), sla.getNamespace()); + } + + for (Element action : (List<Element>) eWfJob.getChildren("action", eWfJob.getNamespace())) { + sla = XmlUtils.getSLAElement(action); + if (sla != null) { + action.removeChildren(sla.getName(), sla.getNamespace()); + } + } + } private String verifySlaElements(Element eWfJob, ELEvaluator evalSla) throws CommandException { String jobSlaXml = ""; // Validate WF job http://git-wip-us.apache.org/repos/asf/oozie/blob/c729cab6/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java index b92510c..5aff988 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java @@ -178,6 +178,15 @@ public class TestSubmitXCommand extends XDataTestCase { conf.set(OozieClient.APP_PATH, workflowUri); conf.set(OozieClient.USER_NAME, getTestUser()); SubmitXCommand sc = new SubmitXCommand(true, conf); + try { + sc.call(); + fail("Should fail with variable cannot be resolved"); + } catch (CommandException ex) { + assertEquals(ErrorCode.E0803, ex.getErrorCode()); + assertEquals("E0803: IO error, variable [foo] cannot be resolved", ex.getMessage()); + } + conf.set("foo", "foo"); + sc = new SubmitXCommand(true, conf); assertEquals("OK", sc.call()); } http://git-wip-us.apache.org/repos/asf/oozie/blob/c729cab6/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index 8f3464c..6500e69 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -77,10 +77,9 @@ public class TestLiteWorkflowAppParser extends XTestCase { " <mapper>/mycat.sh</mapper>\r\n" + " <reducer>/mywc.sh</reducer>\r\n" + " </streaming>\r\n" + - " <job-xml>/tmp</job-xml>\r\n" + " <file>/tmp</file>\r\n" + " <archive>/tmp</archive>\r\n" + - " <job-tracker>foo</job-tracker>\r\n" + + " <job-tracker>${foo}</job-tracker>\r\n" + " <name-node>bar</name-node>\r\n" + " <configuration>\r\n" + " <property>\r\n" + @@ -171,7 +170,7 @@ public class TestLiteWorkflowAppParser extends XTestCase { " <param>x</param>\r\n" + " <file>/tmp</file>\r\n" + " <file>/tmp</file>\r\n" + - " <job-tracker>foo</job-tracker>\r\n" + + " <job-tracker>${foo}</job-tracker>\r\n" + " <name-node>bar</name-node>\r\n" + "</pig>"; e = e.replaceAll(" xmlns=?(\"|\')(\"|\')", ""); http://git-wip-us.apache.org/repos/asf/oozie/blob/c729cab6/core/src/test/resources/wf-schema-valid-global.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-schema-valid-global.xml b/core/src/test/resources/wf-schema-valid-global.xml index 479c80b..735c0b4 100644 --- a/core/src/test/resources/wf-schema-valid-global.xml +++ b/core/src/test/resources/wf-schema-valid-global.xml @@ -17,7 +17,7 @@ --> <workflow-app xmlns="uri:oozie:workflow:0.4" name="test-wf"> <global> - <job-tracker>foo</job-tracker> + <job-tracker>${foo}</job-tracker> <name-node>bar</name-node> <configuration> <property> @@ -58,7 +58,6 @@ <mapper>/mycat.sh</mapper> <reducer>/mywc.sh</reducer> </streaming> - <job-xml>/tmp</job-xml> <file>/tmp</file> <archive>/tmp</archive> </map-reduce> http://git-wip-us.apache.org/repos/asf/oozie/blob/c729cab6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 486b6fa..2248b40 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1618 dryrun should check variable substitution in workflow.xml (bowenzhangusa via rkanter) OOZIE-1681 Sqoop sharelib has no hsqldb jar version (Ostap via rkanter) OOZIE-1691 StackOverflowError in TimestampedMessageParser.parseNextLine() (puru via rkanter) OOZIE-1552 Bring Windows shell script functionality and structure in line with trunk (omaliuvanchuk via rkanter)
