Repository: oozie Updated Branches: refs/heads/master abea46ac3 -> 7e12f0abc
OOZIE-2583 oozie throws EL Exception when reference variable name containing dot (abhishekbafna via jaydeepvishwakarma) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7e12f0ab Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7e12f0ab Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7e12f0ab Branch: refs/heads/master Commit: 7e12f0abc69abe8873dd2d5f7ea4deaf617a5e04 Parents: abea46a Author: jvishwakarma <[email protected]> Authored: Tue Sep 13 17:38:48 2016 +0530 Committer: jvishwakarma <[email protected]> Committed: Tue Sep 13 17:38:48 2016 +0530 ---------------------------------------------------------------------- .../apache/oozie/command/wf/SubmitXCommand.java | 24 ++++++ .../oozie/command/wf/TestSubmitXCommand.java | 82 +++++++++++++++++++- release-log.txt | 1 + 3 files changed, 105 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java index 27274b9..3e2996f 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java @@ -160,6 +160,9 @@ public class SubmitXCommand extends WorkflowXCommand<String> { throw new IOException("default configuration file, " + ex.getMessage(), ex); } } + if (defaultConf != null) { + defaultConf = resolveDefaultConfVariables(defaultConf); + } WorkflowApp app = wps.parseDef(conf, defaultConf); XConfiguration protoActionConf = wps.createProtoActionConf(conf, true); @@ -278,6 +281,27 @@ public class SubmitXCommand extends WorkflowXCommand<String> { } } + /** + * Resolving variables from config-default, which might be referencing into conf/defaultConf + * @param defaultConf config-default.xml + * @return resolved config-default configuration. + */ + private Configuration resolveDefaultConfVariables(Configuration defaultConf) { + XConfiguration resolveDefaultConf = new XConfiguration(); + for (Map.Entry<String, String> entry : defaultConf) { + String defaultConfKey = entry.getKey(); + String defaultConfValue = entry.getValue(); + // if value is referencing some other key, first check within the default config to resolve, + // then job.properties (conf) + if (defaultConfValue.contains("$") && defaultConf.get(defaultConfKey).contains("$")) { + resolveDefaultConf.set(defaultConfKey, conf.get(defaultConfKey)); + } else { + resolveDefaultConf.set(defaultConfKey, defaultConf.get(defaultConfKey)); + } + } + return resolveDefaultConf; + } + private void removeSlaElements(Element eWfJob) { Element sla = XmlUtils.getSLAElement(eWfJob); if (sla != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java index 73464c8..3c893d0 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java @@ -19,14 +19,19 @@ package org.apache.oozie.command.wf; import java.io.File; +import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringReader; import java.net.URI; + import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; +import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor; import org.apache.oozie.local.LocalOozie; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowJob; @@ -41,6 +46,8 @@ import org.apache.oozie.test.XDataTestCase; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.service.XLogService; import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; public class TestSubmitXCommand extends XDataTestCase { @Override @@ -260,7 +267,7 @@ public class TestSubmitXCommand extends XDataTestCase { waitFor(15 * 1000, new Predicate() { public boolean evaluate() throws Exception { - return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED; + return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.PREP; } }); WorkflowJobBean wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId); @@ -283,7 +290,7 @@ public class TestSubmitXCommand extends XDataTestCase { waitFor(15 * 1000, new Predicate() { public boolean evaluate() throws Exception { - return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED; + return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.PREP; } }); wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId1); @@ -297,6 +304,77 @@ public class TestSubmitXCommand extends XDataTestCase { assertNotNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST)); } + public void testWFConfigDefaultVarResolve() throws Exception { + final OozieClient wfClient = LocalOozie.getClient(); + OutputStream os = new FileOutputStream(getTestCaseDir() + "/config-default.xml"); + XConfiguration defaultConf = new XConfiguration(); + defaultConf.set("outputDir", "default-output-dir"); + defaultConf.set("foo.bar", "default-foo-bar"); + defaultConf.set("foobarRef", "${foo.bar}"); + defaultConf.set("key", "default_value"); + defaultConf.set("should_resolve", "${should.resolve}"); + defaultConf.set("mixed", "${nameNode}/${outputDir}"); + defaultConf.writeXml(os); + os.close(); + + String workflowUri = getTestCaseFileUri("workflow.xml"); + String actionXml = "<map-reduce>" + + "<job-tracker>${jobTracker}</job-tracker>" + + "<name-node>${nameNode}</name-node>" + + " <prepare>" + + " <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>" + + " </prepare>" + + " <configuration>" + + " <property><name>bb</name><value>BB</value></property>" + + " <property><name>cc</name><value>from_action</value></property>" + + " </configuration>" + + " </map-reduce>"; + String wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">" + + " <start to=\"mr-node\"/>" + + " <action name=\"mr-node\">" + + actionXml + + " <ok to=\"end\"/>" + + " <error to=\"fail\"/>" + + "</action>" + + "<kill name=\"fail\">" + + " <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + + "</kill>" + + "<end name=\"end\"/>" + + "</workflow-app>"; + + writeToFile(wfXml, workflowUri); + Configuration conf = new XConfiguration(); + conf.set("nameNode", getNameNodeUri()); + conf.set("jobTracker", getJobTrackerUri()); + conf.set("foobarRef", "foobarRef"); + conf.set("key", "job_prop_value"); + conf.set(OozieClient.APP_PATH, workflowUri); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("should.resolve", "resolved"); + SubmitXCommand sc = new SubmitXCommand(conf); + final String jobId = sc.call(); + new StartXCommand(jobId).call(); + waitFor(15 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.KILLED; + } + }); + String actionId = jobId + "@mr-node"; + WorkflowActionBean action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery + .GET_ACTION, actionId); + Element eAction = XmlUtils.parseXml(action.getConf()); + Element eConf = eAction.getChild("configuration", eAction.getNamespace()); + Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString())); + assertEquals("default-output-dir", actionConf.get("outputDir")); + assertEquals("BB", actionConf.get("bb")); + assertEquals("from_action", actionConf.get("cc")); + assertEquals("resolved", actionConf.get("should_resolve")); + assertEquals("default-foo-bar", actionConf.get("foo.bar")); + assertEquals("default-foo-bar", actionConf.get("foobarRef")); + assertEquals("default_value", actionConf.get("key")); + assertEquals(getNameNodeUri()+"/default-output-dir", actionConf.get("mixed")); + } + private void writeToFile(String appXml, String appPath) throws IOException { // TODO Auto-generated method stub File wf = new File(URI.create(appPath)); http://git-wip-us.apache.org/repos/asf/oozie/blob/7e12f0ab/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 8e6e791..c03a5c7 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2583 oozie throws EL Exception when reference variable name containing dot (abhishekbafna via jaydeepvishwakarma) OOZIE-2653 Close active connection to hcat server in fs action (satishsaley via puru) OOZIE-2617 Read properties files in action configurations (wutaklon via jaydeepvishwakarma) OOZIE-2615 Flaky tests TestCoordActionsKillXCommand.testActionKillCommandActionNumbers and testActionKillCommandDate (pbacsko via rkanter)
