http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 879bfeb..794ad81 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -21,6 +21,7 @@ package org.apache.oozie.action.hadoop; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -35,18 +36,22 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.action.ActionExecutor; @@ -55,6 +60,7 @@ import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; @@ -62,7 +68,6 @@ import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.UUIDService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; -import org.apache.oozie.util.FSUtils; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; @@ -336,7 +341,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { return new Context(wf, action); } - protected RunningJob submitAction(Context context, JavaActionExecutor javaActionExecutor) throws Exception { + // TODO: OYA: void + protected RunningJob submitAction(Context context, JavaActionExecutor javaActionExecutor) throws ActionExecutorException { WorkflowAction action = context.getAction(); javaActionExecutor.prepareActionDir(getFileSystem(), context); @@ -348,21 +354,37 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertNotNull(jobId); assertNotNull(jobTracker); assertNotNull(consoleUrl); - - JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); - jobConf.set("yarn.resourcemanager.address", jobTracker); - - JobClient jobClient = - Services.get().get(HadoopAccessorService.class).createJobClient(getTestUser(), jobConf); - final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId)); - assertNotNull(runningJob); - return runningJob; + return null; } - protected RunningJob submitAction(Context context) throws Exception { + // TODO: OYA: void + protected RunningJob submitAction(Context context) throws ActionExecutorException { return submitAction(context, new JavaActionExecutor()); } + private void waitUntilYarnAppState(String externalId, final YarnApplicationState state) + throws HadoopAccessorException, IOException, YarnException { + final ApplicationId appId = ConverterUtils.toApplicationId(externalId); + + JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri()); + // This is needed here because we need a mutable final YarnClient + final MutableObject<YarnClient> yarnClientMO = new MutableObject<YarnClient>(null); + try { + yarnClientMO.setValue(Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), jobConf)); + waitFor(60 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return yarnClientMO.getValue().getApplicationReport(appId).getYarnApplicationState().equals(state); + } + }); + } finally { + if (yarnClientMO.getValue() != null) { + yarnClientMO.getValue().close(); + } + } + assertTrue(yarnClientMO.getValue().getApplicationReport(appId).getYarnApplicationState().equals(state)); + } + public void testSimpestSleSubmitOK() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + @@ -370,14 +392,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); - final RunningJob runningJob = submitAction(context); - waitFor(60 * 1000, new Predicate() { - @Override - public boolean evaluate() throws Exception { - return runningJob.isComplete(); - } - }); - assertTrue(runningJob.isSuccessful()); + submitAction(context); + waitUntilYarnAppState(context.getAction().getExternalId(), YarnApplicationState.FINISHED); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); @@ -1817,118 +1833,12 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { JavaActionExecutor jae = new JavaActionExecutor("java"); jae.setupLauncherConf(conf, xml, appPath, createContext("<java/>", null)); assertEquals(5, conf.size()); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); assertEquals("v1", conf.get("oozie.launcher.p1")); assertEquals("v1", conf.get("p1")); assertEquals("v2b", conf.get("oozie.launcher.p2")); assertEquals("v2b", conf.get("p2")); } - public void testInjectLauncherUseUberMode() throws Exception { - // default -- should set to true - JavaActionExecutor jae = new JavaActionExecutor(); - Configuration conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - jae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to true -- should keep at true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", true); - jae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to false -- should keep at false - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", false); - jae.injectLauncherUseUberMode(conf); - assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - - // disable at oozie-site level for just the "test" action - ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", false); - JavaActionExecutor tjae = new JavaActionExecutor("test"); - - // default -- should not set - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - tjae.injectLauncherUseUberMode(conf); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - // default -- should be true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - jae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to true -- should keep at true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", true); - tjae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - // action conf set to true -- should keep at true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", true); - jae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to false -- should keep at false - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", false); - tjae.injectLauncherUseUberMode(conf); - assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - // action conf set to false -- should keep at false - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", false); - jae.injectLauncherUseUberMode(conf); - assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - - // disable at oozie-site level for all actions except for the "test" action - ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", true); - ConfigurationService.setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", false); - - // default -- should be true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - tjae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - // default -- should not set - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - jae.injectLauncherUseUberMode(conf); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to true -- should keep at true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", true); - tjae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - // action conf set to true -- should keep at true - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", true); - jae.injectLauncherUseUberMode(conf); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - - // action conf set to false -- should keep at false - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", false); - tjae.injectLauncherUseUberMode(conf); - assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - // action conf set to false -- should keep at false - conf = new Configuration(false); - assertNull(conf.get("mapreduce.job.ubertask.enable")); - conf.setBoolean("mapreduce.job.ubertask.enable", false); - jae.injectLauncherUseUberMode(conf); - assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); - } - public void testUpdateConfForJavaTmpDir() throws Exception { //Test UpdateCOnfForJavaTmpDir for launcherConf @@ -1993,355 +1903,6 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS)); assertEquals("-Xmx1024m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); } - public void testUpdateConfForUberMode() throws Exception { - Element actionXml1 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>2048</value></property>" - + "<property><name>oozie.launcher.mapred.child.java.opts</name>" - + "<value>-Xmx2048m -Djava.net.preferIPv4Stack=true</value></property>" - + "<property><name>oozie.launcher.mapred.child.env</name><value>A=foo</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - JavaActionExecutor ae = new JavaActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); - // memoryMB (2048 + 512) - assertEquals("2560", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB)); - // heap size in child.opts (2048 + 512) - int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", - launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true", - launcherConf.get("mapreduce.map.java.opts")); - - assertEquals("-Xmx1024m -Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx2560m -Djava.io.tmpdir=./tmp", - launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - - assertEquals(2560, heapSize); - - // env - assertEquals("A=foo", launcherConf.get(JavaActionExecutor.YARN_AM_ENV)); - - Element actionXml2 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.resource.mb</name><value>3072</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>2048</value></property>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>" - + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>" - + "<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.java.opts</name>" - + "<value>-Xmx2560m -XX:NewRatio=8</value></property>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name><value>A=foo</value></property>" - + "<property><name>oozie.launcher.mapred.child.env</name><value>B=bar</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf); - - // memoryMB (3072 + 512) - assertEquals("3584", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB)); - - // heap size (2560 + 512) - heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts")); - assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx2560m -XX:NewRatio=8 " + - "-Xmx3072m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - assertEquals(3072, heapSize); - - // env (equqls to mapreduce.map.env + am.env) - assertTrue(launcherConf.get(JavaActionExecutor.YARN_AM_ENV).trim().equals("A=foo,B=bar")); - - // Test limit is applied in case of 32 bit - Element actionXml3 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.resource.mb</name><value>3072</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.memory.mb</name><value>4000</value></property>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>" - + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true</value></property>" - + "<property><name>oozie.launcher.mapred.child.java.opts</name><value>-Xmx1536m</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.java.opts</name>" - + "<value>-Xmx4000m -XX:NewRatio=8</value></property>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name><value>A=foo</value></property>" - + "<property><name>oozie.launcher.mapred.child.env</name><value>B=bar</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - - launcherConf = ae.createBaseHadoopConf(context, actionXml3); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf); - - // memoryMB (limit to 4096) - assertEquals("4096", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB)); - - // heap size (limit to 3584) - heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts")); - assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx4000m -XX:NewRatio=8 " + - "-Xmx3584m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - assertEquals(3584, heapSize); - - // env (equqls to mapreduce.map.env + am.env) - assertEquals("A=foo,B=bar", launcherConf.get(JavaActionExecutor.YARN_AM_ENV)); - } - - public void testUpdateConfForUberModeWithEnvDup() throws Exception { - Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>" - + "<value>JAVA_HOME=/home/blah/java/jdk64/current,A=foo,B=bar</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.env</name>" - + "<value>JAVA_HOME=/home/blah/java/jdk64/latest,C=blah</value></property>" + "</configuration>" - + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - JavaActionExecutor ae = new JavaActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); - - // uber mode should be disabled since JAVA_HOME points to different paths in am.evn and map.env - assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - - // testing complicated env setting case - Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property>" - + "<name>oozie.launcher.yarn.app.mapreduce.am.env</name>" - + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_1/lib/native/`$JAVA_HOME/bin/java -d32 -version;" - + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>" - + "<property>" + "<name>oozie.launcher.mapreduce.map.env</name>" - + "<value>LD_LIBRARY_PATH=$HADOOP_HOME_2/lib/native/`$JAVA_HOME/bin/java -d32 -version;" - + "if [ $? -eq 0 ]; then echo Linux-i386-32; else echo Linux-amd64-64;fi`</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf); - - // uber mode should be disabled since LD_LIBRARY_PATH is different in am.evn and map.env - assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - - Element actionXml3 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.env</name>" - + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B</value></property>" - + "<property><name>oozie.launcher.mapreduce.map.env</name>" - + "<value>JAVA_HOME=/home/blah/java/jdk64/current,PATH=A</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - - launcherConf = ae.createBaseHadoopConf(context, actionXml3); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf); - - // uber mode should be enabled since JAVA_HOME is the same, and PATH doesn't conflict - assertEquals("true", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - - // JAVA_HOME, PATH=A duplication is removed - String a = launcherConf.get(JavaActionExecutor.YARN_AM_ENV); - assertEquals("JAVA_HOME=/home/blah/java/jdk64/current,PATH=A,PATH=B", - launcherConf.get(JavaActionExecutor.YARN_AM_ENV)); - } - - public void testUpdateConfForUberModeForJavaOpts() throws Exception { - Element actionXml1 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>" - + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>" - + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx1536m</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" - + "<java-opt>-Xmx2048m</java-opt>" - + "<java-opt>-Dkey1=val1</java-opt>" - + "<java-opt>-Dkey2=val2</java-opt>" - + "</java>"); - JavaActionExecutor ae = new JavaActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); - - // heap size (2048 + 512) - int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2", - launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2", - launcherConf.get("mapreduce.map.java.opts")); - assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Xmx2560m " + - "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - assertEquals(2560, heapSize); - - Element actionXml2 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>" - + "<value>-Xmx1024m -Djava.net.preferIPv4Stack=true </value></property>" - + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx1536m</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" - + "<java-opts>-Xmx2048m -Dkey1=val1</java-opts>" - + "</java>"); - - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf); - - // heap size (2048 + 512) - heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1", - launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1", - launcherConf.get("mapreduce.map.java.opts")); - assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Xmx2560m " + - "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - assertEquals(2560, heapSize); - - Element actionXml3 = XmlUtils - .parseXml("<java>" - + "<job-tracker>" - + getJobTrackerUri() - + "</job-tracker>" - + "<name-node>" - + getNameNodeUri() - + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.yarn.app.mapreduce.am.command-opts</name>" - + "<value>-Xmx2048m -Djava.net.preferIPv4Stack=true </value></property>" - + "<property><name>oozie.launcher.mapreduce.map.java.opts</name><value>-Xmx3072m</value></property>" - + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" - + "<java-opts>-Xmx1024m -Dkey1=val1</java-opts>" - + "</java>"); - - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml3, launcherConf); - - // heap size (2048 + 512) - heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS)); - assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1", - launcherConf.get("mapred.child.java.opts")); - assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1", - launcherConf.get("mapreduce.map.java.opts")); - assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Xmx2560m " + - "-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim()); - assertEquals(2560, heapSize); - } - - public void testDisableUberForProperties() throws Exception { - Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.mapreduce.job.classloader</name>" - + "<value>true</value></property>" - + "</configuration>" - + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - JavaActionExecutor ae = new JavaActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); - - // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true - assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - - Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.mapreduce.user.classpath.first</name>" - + "<value>true</value></property>" - + "</configuration>" - + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - ae = new JavaActionExecutor(); - protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - wf = createBaseWorkflow(protoConf, "action"); - action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - context = new Context(wf, action); - launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf); - - // uber mode should be disabled since oozie.launcher.mapreduce.user.classpath.first=true - assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - } - - public void testDisableUberForUserProperties() throws Exception { - Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" - + "<configuration>" - + "<property><name>oozie.launcher.mapreduce.job.ubertask.enable</name>" - + "<value>false</value></property>" - + "</configuration>" - + "<main-class>MAIN-CLASS</main-class>" + "</java>"); - JavaActionExecutor ae = new JavaActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); - // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true - assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); - } public void testUpdateConfForTimeLineServiceEnabled() throws Exception { Element actionXml = XmlUtils @@ -2766,12 +2327,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("AA", conf.get("a")); assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo")); assertEquals("action.barbar", conf.get("action.foofoo")); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - if (conf.size() == 7) { - assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address")); - } else { - assertEquals(6, conf.size()); - } + assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address")); + assertEquals(6, conf.size()); conf = new Configuration(false); Assert.assertEquals(0, conf.size()); @@ -2780,12 +2337,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals(getJobTrackerUri(), conf.get("yarn.resourcemanager.address")); assertEquals("action.barbar", conf.get("oozie.launcher.action.foofoo")); assertEquals("action.barbar", conf.get("action.foofoo")); - assertEquals("true", conf.get("mapreduce.job.ubertask.enable")); - if (conf.size() == 5) { - assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address")); - } else { - assertEquals(4, conf.size()); - } + assertEquals(getJobTrackerUri(), conf.get("mapreduce.jobtracker.address")); + assertEquals(4, conf.size()); } public void testSetRootLoggerLevel() throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java new file mode 100644 index 0000000..ed29299 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.oozie.service.HadoopAccessorService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.XConfiguration; + +import java.io.File; +import java.io.FileWriter; +import java.io.Writer; +import java.net.URI; +import java.util.Map; + +public class TestLauncherAM extends XFsTestCase { + + + // TODO: OYA: write tests later + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java new file mode 100644 index 0000000..d0b4d5b --- /dev/null +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMCallbackNotifier.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.oozie.QueryServlet; +import org.apache.oozie.command.wf.HangServlet; +import org.apache.oozie.test.EmbeddedServletContainer; +import org.apache.oozie.test.XTestCase; +import org.junit.Assert; +import org.mockito.Mockito; + +import java.net.Proxy; +import java.util.HashMap; +import java.util.Map; + +// A lot of this adapted from org.apache.hadoop.mapreduce.v2.app.TestJobEndNotifier and org.apache.hadoop.mapred.TestJobEndNotifier +public class TestLauncherAMCallbackNotifier extends XTestCase { + + public void testConfiguration() throws Exception { + Configuration conf = new Configuration(false); + + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "0"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "10"); + LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf); + assertEquals(0, cn.numTries); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(1, cn.numTries); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "20"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(11, cn.numTries); //11 because number of _retries_ is 10 + + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(1000, cn.waitInterval); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "10000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(5000, cn.waitInterval); + //Test negative numbers are set to default + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "-10"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(5000, cn.waitInterval); + + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_TIMEOUT, "1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(1000, cn.timeout); + + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(Proxy.Type.DIRECT, cn.proxyToUse.type()); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost:someport"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals(Proxy.Type.DIRECT, cn.proxyToUse.type()); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "somehost:1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals("HTTP @ somehost:1000", cn.proxyToUse.toString()); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "socks@somehost:1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals("SOCKS @ somehost:1000", cn.proxyToUse.toString()); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "SOCKS@somehost:1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals("SOCKS @ somehost:1000", cn.proxyToUse.toString()); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_PROXY, "sfafn@somehost:1000"); + cn = new LauncherAMCallbackNotifier(conf); + assertEquals("HTTP @ somehost:1000", cn.proxyToUse.toString()); + } + + public void testNotifyRetries() throws InterruptedException { + Configuration conf = new Configuration(false); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, "http://nonexistent";); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000"); + + LauncherAMCallbackNotifier cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf)); + long start = System.currentTimeMillis(); + cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED); + long end = System.currentTimeMillis(); + Mockito.verify(cnSpy, Mockito.times(1)).notifyURLOnce(); + Assert.assertTrue("Should have taken more than 5 seconds but it only took " + (end - start), end - start >= 5000); + + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "3"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "3"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "3000"); + + cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf)); + start = System.currentTimeMillis(); + cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED); + end = System.currentTimeMillis(); + Mockito.verify(cnSpy, Mockito.times(3)).notifyURLOnce(); + Assert.assertTrue("Should have taken more than 9 seconds but it only took " + (end - start), end - start >= 9000); + } + + public void testNotifyTimeout() throws Exception { + EmbeddedServletContainer container = null; + try { + container = new EmbeddedServletContainer("blah"); + Map<String, String> params = new HashMap<String, String>(); + params.put(HangServlet.SLEEP_TIME_MS, "1000000"); + container.addServletEndpoint("/hang/*", HangServlet.class, params); + container.start(); + + Configuration conf = new Configuration(false); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, container.getServletURL("/hang/*")); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000"); + + LauncherAMCallbackNotifier cnSpy = Mockito.spy(new LauncherAMCallbackNotifier(conf)); + long start = System.currentTimeMillis(); + cnSpy.notifyURL(FinalApplicationStatus.SUCCEEDED); + long end = System.currentTimeMillis(); + Mockito.verify(cnSpy, Mockito.times(1)).notifyURLOnce(); + Assert.assertTrue("Should have taken more than 5 seconds but it only took " + (end - start), end - start >= 5000); + } finally { + if (container != null) { + container.stop(); + } + } + } + + public void testNotify() throws Exception { + EmbeddedServletContainer container = null; + try { + container = new EmbeddedServletContainer("blah"); + container.addServletEndpoint("/count/*", QueryServlet.class); + container.start(); + + Configuration conf = new Configuration(false); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_ATTEMPTS, "0"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_MAX_ATTEMPTS, "1"); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, container.getServletURL("/count/?status=$jobStatus")); + conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_RETRY_INTERVAL, "5000"); + + LauncherAMCallbackNotifier cn = new LauncherAMCallbackNotifier(conf); + QueryServlet.lastQueryString = null; + assertNull(QueryServlet.lastQueryString); + cn.notifyURL(FinalApplicationStatus.SUCCEEDED); + waitFor(5000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return "status=SUCCEEDED".equals(QueryServlet.lastQueryString); + } + }); + assertEquals("status=SUCCEEDED", QueryServlet.lastQueryString); + } finally { + if (container != null) { + container.stop(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java index df9e939..e940d39 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java @@ -24,6 +24,9 @@ import org.apache.oozie.service.Services; import org.apache.oozie.test.XFsTestCase; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; public class TestPrepareActionsDriver extends XFsTestCase { @@ -40,7 +43,7 @@ public class TestPrepareActionsDriver extends XFsTestCase { } // Test to check if prepare action is performed as expected when the prepare XML block is a valid one - public void testDoOperationsWithValidXML() throws LauncherException, IOException { + public void testDoOperationsWithValidXML() throws LauncherException, IOException, ParserConfigurationException, SAXException { Path actionDir = getFsTestCaseDir(); FileSystem fs = getFileSystem(); Path newDir = new Path(actionDir, "newDir"); @@ -57,7 +60,7 @@ public class TestPrepareActionsDriver extends XFsTestCase { assertTrue(fs.exists(actionDir)); } - // Test to check if LauncherException is thrown when the prepare XML block is invalid + // Test to check if Exception is thrown when the prepare XML block is invalid public void testDoOperationsWithInvalidXML() throws LauncherException, IOException { Path actionDir = getFsTestCaseDir(); FileSystem fs = getFileSystem(); @@ -75,11 +78,9 @@ public class TestPrepareActionsDriver extends XFsTestCase { LauncherMapperHelper.setupLauncherURIHandlerConf(conf); PrepareActionsDriver.doOperations(prepareXML, conf); fail("Expected to catch an exception but did not encounter any"); - } catch (LauncherException le) { - assertEquals(le.getCause().getClass(), org.xml.sax.SAXParseException.class); - assertEquals(le.getMessage(), "Content is not allowed in prolog."); - } catch(Exception ex){ - fail("Expected a LauncherException but received an Exception"); + } catch (Exception ex) { + assertEquals(ex.getClass(), org.xml.sax.SAXParseException.class); + assertEquals(ex.getMessage(), "Content is not allowed in prolog."); } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index 6a962a1..9468fad 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -367,25 +367,4 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { assertNotNull(runningJob); return runningJob; } - - public void testShellMainPathInUber() throws Exception { - Services.get().getConf().setBoolean("oozie.action.shell.launcher.mapreduce.job.ubertask.enable", true); - - Element actionXml = XmlUtils.parseXml("<shell>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" - + "<name-node>" + getNameNodeUri() + "</name-node>" + "<exec>script.sh</exec>" - + "<argument>a=A</argument>" + "<argument>b=B</argument>" + "</shell>"); - ShellActionExecutor ae = new ShellActionExecutor(); - XConfiguration protoConf = new XConfiguration(); - protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); - - WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); - WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); - action.setType(ae.getType()); - - Context context = new Context(wf, action); - JobConf launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, launcherConf); - // env - assertEquals("PATH=.:$PATH", launcherConf.get(JavaActionExecutor.YARN_AM_ENV)); - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java b/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java index 3344cf9..d90aeb6 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java +++ b/core/src/test/java/org/apache/oozie/command/wf/HangServlet.java @@ -18,6 +18,8 @@ package org.apache.oozie.command.wf; +import org.apache.oozie.util.XLog; + import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -25,14 +27,27 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; /** - * Servlet that 'hangs' for 200 ms. Used by TestNotificationXCommand + * Servlet that 'hangs' for some amount of time (200ms) by default. + * The time can be configured by setting {@link HangServlet#SLEEP_TIME_MS} as an init parameter for the servlet. */ public class HangServlet extends HttpServlet { + public static final String SLEEP_TIME_MS = "sleep_time_ms"; + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { try { - Thread.sleep(200); + long time = 200; + String sleeptime = getInitParameter(SLEEP_TIME_MS); + if (sleeptime != null) { + try { + time = Long.parseLong(sleeptime); + } catch (NumberFormatException nfe) { + XLog.getLog(HangServlet.class).error("Invalid sleep time, using default (200)", nfe); + } + } + XLog.getLog(HangServlet.class).info("Sleeping for " + time + " ms"); + Thread.sleep(time); } catch (Exception ex) { //NOP http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java index 2153bf1..9b48df5 100644 --- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java @@ -209,9 +209,6 @@ public class TestConfigurationService extends XTestCase { assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA)); assertEquals("http://localhost:8080/oozie?job=";, ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL)); - assertEquals(true, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE)); - assertEquals(false, ConfigurationService.getBoolean( - "oozie.action.shell.launcher." + JavaActionExecutor.HADOOP_YARN_UBER_MODE)); assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED)); assertEquals(0, ConfigurationService.getStrings("no.defined").length); http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java index 96faa48..2798719 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java @@ -18,7 +18,17 @@ package org.apache.oozie.service; -import org.apache.oozie.test.XTestCase; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.authorize.*; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.oozie.test.XFsTestCase; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.fs.FileSystem; @@ -34,7 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; import org.apache.oozie.util.XConfiguration; -public class TestHadoopAccessorService extends XTestCase { +public class TestHadoopAccessorService extends XFsTestCase { protected void setUp() throws Exception { super.setUp(); @@ -136,45 +146,89 @@ public class TestHadoopAccessorService extends XTestCase { */ assertEquals("100", conf.get("action.testprop")); assertEquals("1", conf.get("default.testprop")); + } + + public void testCreateJobClient() throws Exception { + HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + JobConf conf = has.createJobConf(getJobTrackerUri()); + + JobClient jc = has.createJobClient(getTestUser(), conf); + assertNotNull(jc); + jc.getAllJobs(); + + try { + has.createJobClient("invalid-user", conf); + fail("Should have thrown exception because not allowed to impersonate 'invalid-user'"); + } + catch (HadoopAccessorException ex) { + assertEquals(ErrorCode.E0902, ex.getErrorCode()); + } + JobConf conf2 = new JobConf(false); + conf2.set("mapred.job.tracker", getJobTrackerUri()); + try { + has.createJobClient(getTestUser(), conf2); + fail("Should have thrown exception because Configuration not created by HadoopAccessorService"); + } + catch (HadoopAccessorException ex) { + assertEquals(ErrorCode.E0903, ex.getErrorCode()); + } } - public void testAccessor() throws Exception { - Services services = Services.get(); - HadoopAccessorService has = services.get(HadoopAccessorService.class); + public void testCreateYarnClient() throws Exception { + HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); JobConf conf = has.createJobConf(getJobTrackerUri()); - conf.set("mapred.job.tracker", getJobTrackerUri()); - conf.set("fs.default.name", getNameNodeUri()); - URI uri = new URI(getNameNodeUri()); + YarnClient yc = has.createYarnClient(getTestUser(), conf); + assertNotNull(yc); + yc.getApplications(); - //valid user - String user = getTestUser(); - String group = getTestGroup(); + try { + yc = has.createYarnClient("invalid-user", conf); + assertNotNull(yc); + yc.getApplications(); + fail("Should have thrown exception because not allowed to impersonate 'invalid-user'"); + } + catch (AuthorizationException ex) { + } - JobClient jc = has.createJobClient(user, conf); - assertNotNull(jc); - FileSystem fs = has.createFileSystem(user, new URI(getNameNodeUri()), conf); - assertNotNull(fs); - fs = has.createFileSystem(user, uri, conf); - assertNotNull(fs); + JobConf conf2 = new JobConf(false); + conf2.set("yarn.resourcemanager.address", getJobTrackerUri()); + try { + has.createYarnClient(getTestUser(), conf2); + fail("Should have thrown exception because Configuration not created by HadoopAccessorService"); + } + catch (HadoopAccessorException ex) { + assertEquals(ErrorCode.E0903, ex.getErrorCode()); + } + } - //invalid user + public void testCreateFileSystem() throws Exception { + HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + JobConf conf = has.createJobConf(getJobTrackerUri()); - user = "invalid"; + FileSystem fs = has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), conf); + assertNotNull(fs); + fs.exists(new Path(getNameNodeUri(), "/foo")); try { - has.createJobClient(user, conf); - fail(); + fs = has.createFileSystem("invalid-user", new URI(getNameNodeUri()), conf); + assertNotNull(fs); + fs.exists(new Path(getNameNodeUri(), "/foo")); + fail("Should have thrown exception because not allowed to impersonate 'invalid-user'"); } - catch (Throwable ex) { + catch (RemoteException ex) { + assertEquals(AuthorizationException.class.getName(), ex.getClassName()); } + JobConf conf2 = new JobConf(false); + conf2.set("fs.default.name", getNameNodeUri()); try { - has.createFileSystem(user, uri, conf); - fail(); + has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), conf2); + fail("Should have thrown exception because Configuration not created by HadoopAccessorService"); } - catch (Throwable ex) { + catch (HadoopAccessorException ex) { + assertEquals(ErrorCode.E0903, ex.getErrorCode()); } } @@ -290,4 +344,21 @@ public class TestHadoopAccessorService extends XTestCase { } has.destroy(); } + + public void testCreateLocalResourceForConfigurationFile() throws Exception { + HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); + String filename = "foo.xml"; + Configuration conf = has.createJobConf(getNameNodeUri()); + conf.set("foo", "bar"); + LocalResource lRes = has.createLocalResourceForConfigurationFile(filename, getTestUser(), conf, getFileSystem().getUri(), + getFsTestCaseDir()); + assertNotNull(lRes); + assertEquals(LocalResourceType.FILE, lRes.getType()); + assertEquals(LocalResourceVisibility.APPLICATION, lRes.getVisibility()); + Path resPath = ConverterUtils.getPathFromYarnURL(lRes.getResource()); + assertEquals(new Path(getFsTestCaseDir(), "foo.xml"), resPath); + Configuration conf2 = new Configuration(false); + conf2.addResource(getFileSystem().open(resPath)); + assertEquals("bar", conf2.get("foo")); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index e360369..81a33fd 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -82,6 +82,7 @@ import org.apache.oozie.sla.SLASummaryBean; import org.apache.oozie.store.StoreException; import org.apache.oozie.test.MiniHCatServer.RUNMODE; import org.apache.oozie.test.hive.MiniHS2; +import org.apache.oozie.util.ClasspathUtils; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.ParamChecker; import org.apache.oozie.util.XConfiguration; @@ -877,6 +878,7 @@ public abstract class XTestCase extends TestCase { private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster2 = null; + // TODO: OYA: replace with MiniYarnCluster or MiniMRYarnCluster private static MiniMRCluster mrCluster = null; private static MiniHCatServer hcatServer = null; private static MiniHS2 hiveserver2 = null; @@ -886,6 +888,8 @@ public abstract class XTestCase extends TestCase { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", testCaseDir); } + // Tell the ClasspathUtils that we're using a mini cluster + ClasspathUtils.setUsingMiniYarnCluster(true); int taskTrackers = 2; int dataNodes = 2; String oozieUser = getOozieUser(); http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java new file mode 100644 index 0000000..3a7215b --- /dev/null +++ b/core/src/test/java/org/apache/oozie/util/TestClasspathUtils.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.util; + +import junit.framework.TestCase; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.test.XFsTestCase; +import org.apache.oozie.test.XTestCase; + +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class TestClasspathUtils extends XFsTestCase { + + @Override + protected void setUp() throws Exception { + super.setUp(); + // This is normally true, and adds the entirety of the current classpath in ClasspathUtils, which we don't want to test or + // worry about here. Temporarily set this back to false so it behaves normally. + ClasspathUtils.setUsingMiniYarnCluster(false); + } + + @Override + protected void tearDown() throws Exception { + // Make sure to turn this back on for subsequent tests + ClasspathUtils.setUsingMiniYarnCluster(true); + super.tearDown(); + } + + public void testSetupClasspath() throws Exception { + Configuration conf = new Configuration(false); + Map<String, String> env = new HashMap<String, String>(); + + Path p1 = new Path(getFsTestCaseDir(), "foo.xml"); + getFileSystem().createNewFile(p1); + DistributedCache.addFileToClassPath(p1, conf); + + Path p2 = new Path(getFsTestCaseDir(), "foo.txt"); + getFileSystem().createNewFile(p2); + DistributedCache.addFileToClassPath(p2, conf); + + Path p3 = new Path(getFsTestCaseDir(), "foo.zip"); + getFileSystem().createNewFile(p3); + DistributedCache.addArchiveToClassPath(p3, conf); + + ClasspathUtils.setupClasspath(env, conf); + + assertEquals(2, env.size()); + assertTrue(env.containsKey("CLASSPATH")); + String[] paths = env.get("CLASSPATH").split(":"); + assertEquals(12, paths.length); + Arrays.sort(paths); + assertEquals("$HADOOP_COMMON_HOME/share/hadoop/common/*", paths[0]); + assertEquals("$HADOOP_COMMON_HOME/share/hadoop/common/lib/*", paths[1]); + assertEquals("$HADOOP_CONF_DIR", paths[2]); + assertEquals("$HADOOP_HDFS_HOME/share/hadoop/hdfs/*", paths[3]); + assertEquals("$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*", paths[4]); + assertEquals("$HADOOP_YARN_HOME/share/hadoop/yarn/*", paths[5]); + assertEquals("$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*", paths[6]); + assertEquals("$PWD", paths[7]); + assertEquals("$PWD/*", paths[8]); + assertEquals("job.jar/classes/", paths[9]); + assertEquals("job.jar/job.jar", paths[10]); + assertEquals("job.jar/lib/*", paths[11]); + + assertTrue(env.containsKey("$PWD")); + paths = env.get("$PWD").split(":"); + assertEquals(3, paths.length); + Arrays.sort(paths); + assertEquals("$PWD/foo.txt", paths[0]); + assertEquals("$PWD/foo.xml", paths[1]); + assertEquals("$PWD/foo.zip", paths[2]); + } + + public void testAddMapReduceToClasspath() throws Exception { + Configuration conf = new Configuration(false); + Map<String, String> env = new HashMap<String, String>(); + + ClasspathUtils.addMapReduceToClasspath(env, conf); + + assertEquals(1, env.size()); + assertTrue(env.containsKey("CLASSPATH")); + String[] paths = env.get("CLASSPATH").split(":"); + assertEquals(2, paths.length); + Arrays.sort(paths); + assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*", paths[0]); + assertEquals("$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*", paths[1]); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3a23cbf..c75911e 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2590 OYA: Create basic Oozie Launcher Application Master (rkanter) OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter) OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru) OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru) http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/distcp/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/distcp/pom.xml b/sharelib/distcp/pom.xml index c8cc47c..cb01faa 100644 --- a/sharelib/distcp/pom.xml +++ b/sharelib/distcp/pom.xml @@ -91,18 +91,6 @@ <outputFile>${project.build.directory}/classpath</outputFile> </configuration> </execution> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate the required classpath - that is required in the env of the launch container in the mini mr/yarn cluster --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hcatalog/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hcatalog/pom.xml b/sharelib/hcatalog/pom.xml index 2b0c504..f4273a5 100644 --- a/sharelib/hcatalog/pom.xml +++ b/sharelib/hcatalog/pom.xml @@ -297,18 +297,6 @@ <outputFile>${project.build.directory}/classpath</outputFile> </configuration> </execution> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate the required classpath - that is required in the env of the launch container in the mini mr/yarn cluster --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml index d10d7b8..ba49403 100644 --- a/sharelib/hive/pom.xml +++ b/sharelib/hive/pom.xml @@ -171,18 +171,6 @@ <outputFile>${project.build.directory}/classpath</outputFile> </configuration> </execution> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate the required classpath - that is required in the env of the launch container in the mini mr/yarn cluster --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/hive2/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml index ce967c5..329832d 100644 --- a/sharelib/hive2/pom.xml +++ b/sharelib/hive2/pom.xml @@ -152,18 +152,6 @@ <outputFile>${project.build.directory}/classpath</outputFile> </configuration> </execution> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate the required classpath - that is required in the env of the launch container in the mini mr/yarn cluster --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> </executions> </plugin> <plugin> http://git-wip-us.apache.org/repos/asf/oozie/blob/fea512cf/sharelib/oozie/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml index dd95b45..b2da4e2 100644 --- a/sharelib/oozie/pom.xml +++ b/sharelib/oozie/pom.xml @@ -85,18 +85,6 @@ <outputFile>${project.build.directory}/classpath</outputFile> </configuration> </execution> - <execution> - <id>create-mrapp-generated-classpath</id> - <phase>generate-test-resources</phase> - <goals> - <goal>build-classpath</goal> - </goals> - <configuration> - <!-- needed to run the unit test for DS to generate the required classpath - that is required in the env of the launch container in the mini mr/yarn cluster --> - <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile> - </configuration> - </execution> </executions> </plugin> <plugin>
