Repository: oozie Updated Branches: refs/heads/master 83dec8b0e -> a0aa6fbdf
OOZIE-1825 Optimize wf_jobs protoconf storage (puru via rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a0aa6fbd Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a0aa6fbd Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a0aa6fbd Branch: refs/heads/master Commit: a0aa6fbdfab47e803d36f8540ca28063907cb472 Parents: 83dec8b Author: Rohini Palaniswamy <[email protected]> Authored: Tue May 13 13:58:48 2014 -0700 Committer: Rohini Palaniswamy <[email protected]> Committed: Tue May 13 13:58:48 2014 -0700 ---------------------------------------------------------------------- .../oozie/service/WorkflowAppService.java | 14 +++--- .../oozie/command/wf/TestSubmitXCommand.java | 52 +++++++++++++++++++- release-log.txt | 1 + 3 files changed, 59 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java index c2f3836..05efefb 100644 --- a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java +++ b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java @@ -151,7 +151,6 @@ public abstract class WorkflowAppService implements Service { throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex); } } - /** * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be * added to distributed cache. These paths include .jar,.so and the resource file paths. @@ -168,9 +167,12 @@ public abstract class WorkflowAppService implements Service { URI uri = new URI(jobConf.get(OozieClient.APP_PATH)); Configuration conf = has.createJobConf(uri.getAuthority()); + XConfiguration protoConf = new XConfiguration(); + String user = jobConf.get(OozieClient.USER_NAME); conf.set(OozieClient.USER_NAME, user); + protoConf.set(OozieClient.USER_NAME, user); FileSystem fs = has.createFileSystem(user, uri, conf); @@ -227,7 +229,7 @@ public abstract class WorkflowAppService implements Service { } } - conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()])); + protoConf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()])); //Add all properties start with 'oozie.' for (Map.Entry<String, String> entry : jobConf) { @@ -235,14 +237,12 @@ public abstract class WorkflowAppService implements Service { String name = entry.getKey(); String value = entry.getValue(); // if property already exists, should not overwrite - if(conf.get(name) == null) { - conf.set(name, value); + if(protoConf.get(name) == null) { + protoConf.set(name, value); } } } - XConfiguration retConf = new XConfiguration(); - XConfiguration.copy(conf, retConf); - return retConf; + return protoConf; } catch (IOException ex) { throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java index 5aff988..e8f77df 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java @@ -21,15 +21,19 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.io.StringReader; import java.net.URI; - import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.local.LocalOozie; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.command.CommandException; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery; import org.apache.oozie.service.Services; +import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; import org.apache.oozie.store.WorkflowStore; import org.apache.oozie.test.XDataTestCase; @@ -237,6 +241,52 @@ public class TestSubmitXCommand extends XDataTestCase { } } + // It should not store Hadoop properties + public void testProtoConfStorage() throws Exception { + final OozieClient wfClient = LocalOozie.getClient(); + + Configuration conf = new XConfiguration(); + String workflowUri = getTestCaseFileUri("workflow.xml"); + String appXml = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='${appName}-foo'> " + "<start to='end' /> " + + "<end name='end' /> " + "</workflow-app>"; + + writeToFile(appXml, workflowUri); + conf.set(OozieClient.APP_PATH, workflowUri); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("appName", "var-app-name"); + SubmitXCommand sc = new SubmitXCommand(conf); + final String jobId = sc.call(); + + waitFor(15 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED; + } + }); + WorkflowJobBean wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId); + XConfiguration protoConf = new XConfiguration(new StringReader(wf.getProtoActionConf())); + //username, libpath, apppath + assertEquals(protoConf.size(), 2); + assertNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST)); + + new File(getTestCaseDir() + "/lib").mkdirs(); + File.createTempFile("parentLibrary", ".jar", new File(getTestCaseDir() + "/lib")); + conf.set(OozieClient.APP_PATH, workflowUri); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("appName", "var-app-name"); + sc = new SubmitXCommand(conf); + final String jobId1 = sc.call(); + + waitFor(15 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId1).getStatus() == WorkflowJob.Status.SUCCEEDED; + } + }); + wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId1); + protoConf = new XConfiguration(new StringReader(wf.getProtoActionConf())); + assertEquals(protoConf.size(), 3); + assertNotNull(protoConf.get(WorkflowAppService.APP_LIB_PATH_LIST)); + } + private void writeToFile(String appXml, String appPath) throws IOException { // TODO Auto-generated method stub File wf = new File(URI.create(appPath)); http://git-wip-us.apache.org/repos/asf/oozie/blob/a0aa6fbd/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index c35ca7d..f5d0154 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1825 Optimize wf_jobs protoconf storage (puru via rohini) OOZIE-1831 Oozie upgrade fails if workflow jobs are in running or suspended state (satish.mittal via rohini) OOZIE-1690 TestShellActionExecutor#testEnvVar failed for Windows (omaliuvanchuk via rkanter) OOZIE-1243 libtools dir should not include hadoop JARs (satish.mittal via rohini)
