Repository: oozie Updated Branches: refs/heads/master f73303735 -> e365bf9ce
OOZIE-3072 oozie.service.HadoopAccessorService.action.configurations should overwrite default values set in Hadoop's configuration files (gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e365bf9c Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e365bf9c Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e365bf9c Branch: refs/heads/master Commit: e365bf9ce0b1ba55cfcaef3e03c6116777dfdeef Parents: f733037 Author: Gezapeti Cseh <gezap...@apache.org> Authored: Tue Oct 3 13:17:50 2017 +0200 Committer: Gezapeti Cseh <gezap...@apache.org> Committed: Tue Oct 3 13:17:50 2017 +0200 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 4 +- .../action/hadoop/MapReduceActionExecutor.java | 2 +- .../action/hadoop/ActionExecutorTestCase.java | 35 ------- .../action/hadoop/TestJavaActionExecutor.java | 101 ++++++++++++++++++- core/src/test/resources/test-action-config.xml | 5 + release-log.txt | 1 + 6 files changed, 109 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index be05603..76d0daa 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -428,7 +428,7 @@ public class JavaActionExecutor extends ActionExecutor { try { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_YARN_RM), getType()); - XConfiguration.injectDefaults(actionDefaults, actionConf); + XConfiguration.copy(actionDefaults, actionConf); has.checkSupportedFilesystem(appPath.toUri()); // Set the Java Main Class for the Java action to give to the Java launcher @@ -993,8 +993,8 @@ public class JavaActionExecutor extends ActionExecutor { // action job configuration Configuration actionConf = loadHadoopDefaultResources(context, actionXml); - addAppNameContext(action, context); setupActionConf(actionConf, context, actionXml, appPathRoot); + addAppNameContext(action, context); LOG.debug("Setting LibFilesArchives "); setLibFilesArchives(context, actionXml, appPathRoot, actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 22d5526..233dce7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -163,8 +163,8 @@ public class MapReduceActionExecutor extends JavaActionExecutor { regularMR = true; } } - setJobName(actionConf, context); actionConf = super.setupActionConf(actionConf, context, actionXml, appPath); + setJobName(actionConf, context); // For "regular" (not streaming or pipes) MR jobs if (regularMR) { http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java index dd90536..36b3026 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java @@ -298,41 +298,6 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase { return workflow; } - protected WorkflowJobBean createBaseWorkflowWithLauncherConfig(XConfiguration protoConf, String actionName) throws Exception { - Path appUri = new Path(getAppPath(), "workflow.xml"); - - String content = "<workflow-app xmlns='uri:oozie:workflow:1.0' xmlns:sla='uri:oozie:sla:0.1' name='no-op-wf'>"; - content += "<global>" - + "<launcher>" - + " <vcores>2</vcores>" - + " <memory.mb>1024</memory.mb>" - + " <queue>default</queue>" - + " <priority>1</priority>" - + " <java-opts>-verbose:class</java-opts>" - + "</launcher>" - + "</global>"; - - content += "<start to='end' />"; - content += "<end name='end' /></workflow-app>"; - writeToFile(content, getAppPath(), "workflow.xml"); - - WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", - new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, - "end")) - .addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class)); - XConfiguration wfConf = new XConfiguration(); - wfConf.set(OozieClient.USER_NAME, getTestUser()); - wfConf.set(OozieClient.APP_PATH, appUri.toString()); - - WorkflowJobBean workflow = createWorkflow(app, wfConf, protoConf); - - WorkflowActionBean action = new WorkflowActionBean(); - action.setName(actionName); - action.setId(Services.get().get(UUIDService.class).generateChildId(workflow.getId(), actionName)); - workflow.getActions().add(action); - return workflow; - } - private WorkflowJobBean createWorkflow(WorkflowApp app, Configuration conf, XConfiguration protoConf) throws Exception { WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB(); http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 0d1dd97..5c24c24 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -18,9 +18,11 @@ package org.apache.oozie.action.hadoop; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -44,7 +46,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -56,6 +57,7 @@ import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.local.LocalOozie; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; @@ -2181,6 +2183,103 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals(3, conf.size()); } + public void testDefaultConfigurationInActionConf() throws Exception { + JavaActionExecutor ae = new JavaActionExecutor(); + String xmlStr = getJavaActionXml(true); + Element actionXml = XmlUtils.parseXml(xmlStr); + Context context = createContext(xmlStr, getTestGroup()); + Configuration conf = new Configuration(true); + conf.set(YARN_RESOURCEMANAGER_ADDRESS, getJobTrackerUri()); + assertEquals("MapReduce's default value changed", "4", conf.get("mapreduce.map.maxattempts")); + ae.setupActionConf(conf, context, actionXml, new Path(context.getWorkflow().getAppPath())); + assertEquals("failed to inject property >action.foo<","AA", conf.get("action.foo")); + assertEquals("failed to inject property >oozie.launcher.action.foofoo<","action.barbar", + conf.get("oozie.launcher.action.foofoo")); + assertEquals("Maxattempts should've been overwritten by setupActionConf", "1", + conf.get("mapreduce.map.maxattempts")); + } + + private String getJavaActionXml(boolean addConfig) { + String config = addConfig ? "<configuration>" + + "<property><name>action.foo</name><value>AA</value></property>" + + "</configuration>" : ""; + return "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + config + + "<main-class>MAIN-CLASS</main-class>" + + "</java>"; + } + + private String createTestWorkflowXml() throws IOException { + String workflowUri = getTestCaseFileUri("workflow.xml"); + String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.4\" name=\"workflow\">" + + "<global>" + + " <configuration>" + + " <property>" + + " <name>action.foo</name>" + + " <value>foo2</value>" + + " </property>" + + " </configuration>" + + "</global>" + + "<start to=\"java\"/>" + + "<action name=\"java\">" + + getJavaActionXml(false)+ + " <ok to=\"end\"/>" + + " <error to=\"fail\"/>" + + "</action>" + + "<kill name=\"fail\">" + + " <message>Sub workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + + "</kill>" + + "<end name=\"end\"/>" + + "</workflow-app>"; + + final File f = new File(URI.create(workflowUri)); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(appXml.getBytes("UTF-8")); + IOUtils.copyStream(inputStream, new FileOutputStream(f)); + return workflowUri; + } + + public void testGlobalConfigurationWithActionDefaults() throws Exception { + try { + String workflowUri = createTestWorkflowXml(); + LocalOozie.start(); + final OozieClient wfClient = LocalOozie.getClient(); + Properties conf = wfClient.createConfiguration(); + conf.setProperty(OozieClient.APP_PATH, workflowUri); + conf.setProperty(OozieClient.USER_NAME, getTestUser()); + conf.setProperty("appName", "var-app-name"); + final String jobId = wfClient.submit(conf); + wfClient.start(jobId); + WorkflowJob workflow = wfClient.getJobInfo(jobId); + waitFor(20 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + WorkflowAction javaAction = getJavaAction(wfClient.getJobInfo(jobId)); + return javaAction != null && !javaAction.getStatus().equals("PREP"); + } + }); + final WorkflowAction workflowAction = getJavaAction(workflow); + Element eConf = XmlUtils.parseXml(workflowAction.getConf()); + Element element = eConf.getChild("configuration", eConf.getNamespace()); + Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(element).toString())); + + assertEquals("Config value set in <global> section is not propagated correctly", + "foo2", actionConf.get("action.foo")); + } finally { + LocalOozie.stop(); + } + } + + private WorkflowAction getJavaAction(WorkflowJob workflowJob){ + List<WorkflowAction> actions = workflowJob.getActions(); + for(WorkflowAction wa : actions){ + if(wa.getType().equals("java")){ + return wa; + } + } + return null; + } + public void testSetRootLoggerLevel() throws Exception { String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL; String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherAMUtils.ROOT_LOGGER_LEVEL; http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/core/src/test/resources/test-action-config.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/test-action-config.xml b/core/src/test/resources/test-action-config.xml index 6817432..614d477 100644 --- a/core/src/test/resources/test-action-config.xml +++ b/core/src/test/resources/test-action-config.xml @@ -33,4 +33,9 @@ <name>action.testprop</name> <value>100</value> </property> + + <property> + <name>mapreduce.map.maxattempts</name> + <value>1</value> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/oozie/blob/e365bf9c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 69ad0b3..604c072 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-3072 oozie.service.HadoopAccessorService.action.configurations should overwrite default values set in Hadoop's configuration files (gezapeti) OOZIE-2963 Use net.hydromatic instead of org.pentaho aggdesigner-algorithmin pomfilesi (dbist13 via gezapeti) OOZIE-3070 Remove references to org.mortbay.jetty (pbacsko) OOZIE-2885 Running Spark actions should not need Hive on the classpath (satishsaley)
