Author: tucu
Date: Fri Feb 10 23:39:19 2012
New Revision: 1242984
URL: http://svn.apache.org/viewvc?rev=1242984&view=rev
Log:
OOZIE-691 Fix testcases using launcher using Hadoop 0.23.x (tucu)
Modified:
incubator/oozie/trunk/core/pom.xml
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPipesMain.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/pom.xml
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/pom.xml?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
--- incubator/oozie/trunk/core/pom.xml (original)
+++ incubator/oozie/trunk/core/pom.xml Fri Feb 10 23:39:19 2012
@@ -36,7 +36,7 @@
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-hadoop-test</artifactId>
- <scope>test</scope>
+ <scope>compile</scope>
</dependency>
<dependency>
@@ -267,6 +267,23 @@
</configuration>
</plugin>
<plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate the
required classpath
+ that is required in the env of the launch container
in the mini mr/yarn cluster -->
+
<outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Fri Feb 10 23:39:19 2012
@@ -217,6 +217,7 @@ public class JavaActionExecutor extends
checkForDisallowedProps(launcherConf, "inline launcher
configuration");
XConfiguration.copy(launcherConf, conf);
}
+ conf.set("mapreduce.framework.name", "yarn");
return conf;
}
catch (IOException ex) {
@@ -263,6 +264,7 @@ public class JavaActionExecutor extends
checkForDisallowedProps(inlineConf, "inline configuration");
XConfiguration.copy(inlineConf, actionConf);
}
+ actionConf.set("mapreduce.framework.name", "yarn");
return actionConf;
}
catch (IOException ex) {
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
Fri Feb 10 23:39:19 2012
@@ -49,6 +49,7 @@ public class TestLauncher extends XFsTes
Path actionDir = getFsTestCaseDir();
File jar = IOUtils.createJar(new File(getTestCaseDir()),
"launcher.jar", LauncherMapper.class,
+ LauncherMainException.class,
LauncherSecurityManager.class,
LauncherException.class, LauncherMainTester.class);
FileSystem fs = getFileSystem();
@@ -57,12 +58,14 @@ public class TestLauncher extends XFsTes
fs.copyFromLocalFile(new Path(jar.toString()), launcherJar);
JobConf jobConf = new JobConf();
+// jobConf.setJar(jar.getAbsolutePath());
jobConf.set("user.name", getTestUser());
jobConf.set("group.name", getTestGroup());
jobConf.setInt("mapred.map.tasks", 1);
jobConf.setInt("mapred.map.max.attempts", 1);
jobConf.setInt("mapred.reduce.max.attempts", 1);
+ jobConf.set("mapreduce.framework.name", "yarn");
jobConf.set("mapred.job.tracker", getJobTrackerUri());
jobConf.set("fs.default.name", getNameNodeUri());
injectKerberosInfo(jobConf);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionError.java
Fri Feb 10 23:39:19 2012
@@ -205,7 +205,9 @@ public class TestMapReduceActionError ex
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ conf.set("mapreduce.framework.name", "yarn");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
jobConf);
@@ -231,8 +233,9 @@ public class TestMapReduceActionError ex
Configuration conf = ae.createBaseHadoopConf(context,
XmlUtils.parseXml(actionXml));
String user = conf.get("user.name");
String group = conf.get("group.name");
- JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
-
new JobConf(conf));
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
+ JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
jobConf);
final RunningJob mrJob =
jobClient.getJob(JobID.forName(context.getAction().getExternalId()));
waitFor(60 * 1000, new Predicate() {
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
Fri Feb 10 23:39:19 2012
@@ -215,7 +215,9 @@ public class TestMapReduceActionExecutor
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ conf.set("mapreduce.framework.name", "yarn");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
jobConf);
@@ -229,7 +231,7 @@ public class TestMapReduceActionExecutor
Context context = createContext(name, actionXml);
final RunningJob launcherJob = submitAction(context);
String launcherId = context.getAction().getExternalId();
- waitFor(120 * 1000, new Predicate() {
+ waitFor(120 * 2000, new Predicate() {
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
@@ -245,8 +247,10 @@ public class TestMapReduceActionExecutor
Configuration conf = ae.createBaseHadoopConf(context,
XmlUtils.parseXml(actionXml));
String user = conf.get("user.name");
String group = conf.get("group.name");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
- new JobConf(conf));
+ jobConf);
final RunningJob mrJob =
jobClient.getJob(JobID.forName(context.getAction().getExternalId()));
waitFor(120 * 1000, new Predicate() {
@@ -266,7 +270,7 @@ public class TestMapReduceActionExecutor
//hadoop.counters will always be set in case of MR action.
assertNotNull(context.getVar("hadoop.counters"));
String counters = context.getVar("hadoop.counters");
- assertTrue(counters.contains("Task$Counter"));
+ assertTrue(counters.contains("Counter"));
//External Child IDs will always be null in case of MR action.
assertNull(context.getExternalChildIDs());
@@ -293,8 +297,10 @@ public class TestMapReduceActionExecutor
Configuration conf = ae.createBaseHadoopConf(context,
XmlUtils.parseXml(actionXml));
String user = conf.get("user.name");
String group = conf.get("group.name");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
- new JobConf(conf));
+ jobConf);
final RunningJob mrJob =
jobClient.getJob(JobID.forName(context.getAction().getExternalId()));
waitFor(120 * 1000, new Predicate() {
@@ -488,8 +494,10 @@ public class TestMapReduceActionExecutor
Configuration conf = ae.createBaseHadoopConf(context,
XmlUtils.parseXml(actionXml));
String user = conf.get("user.name");
String group = conf.get("group.name");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
JobClient jobClient = Services.get().get(HadoopAccessorService.class)
- .createJobClient(user, group, new JobConf(conf));
+ .createJobClient(user, group, jobConf);
final RunningJob mrJob =
jobClient.getJob(JobID.forName(context.getAction().getExternalId()));
waitFor(120 * 1000, new Predicate() {
@@ -509,9 +517,7 @@ public class TestMapReduceActionExecutor
// Assert for stats info stored in the context.
assertNotNull(context.getExecutionStats());
assertTrue(context.getExecutionStats().contains("ACTION_TYPE"));
-
assertTrue(context.getExecutionStats().contains("JobInProgress$Counter"));
- assertTrue(context.getExecutionStats().contains("FileSystemCounters"));
- assertTrue(context.getExecutionStats().contains("Task$Counter"));
+ assertTrue(context.getExecutionStats().contains("Counter"));
// External Child IDs will always be null in case of MR action.
assertNull(context.getExternalChildIDs());
@@ -519,7 +525,7 @@ public class TestMapReduceActionExecutor
// hadoop.counters will always be set in case of MR action.
assertNotNull(context.getVar("hadoop.counters"));
String counters = context.getVar("hadoop.counters");
- assertTrue(counters.contains("Task$Counter"));
+ assertTrue(counters.contains("Counter"));
}
// Test to assert that executionStats is not set when user has specified
@@ -550,7 +556,7 @@ public class TestMapReduceActionExecutor
Context context = createContext("map-reduce", actionXml);
final RunningJob launcherJob = submitAction(context);
String launcherId = context.getAction().getExternalId();
- waitFor(120 * 1000, new Predicate() {
+ waitFor(120 * 2000, new Predicate() {
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
@@ -566,8 +572,10 @@ public class TestMapReduceActionExecutor
Configuration conf = ae.createBaseHadoopConf(context,
XmlUtils.parseXml(actionXml));
String user = conf.get("user.name");
String group = conf.get("group.name");
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
JobClient jobClient = Services.get().get(HadoopAccessorService.class)
- .createJobClient(user, group, new JobConf(conf));
+ .createJobClient(user, group, jobConf);
final RunningJob mrJob =
jobClient.getJob(JobID.forName(context.getAction().getExternalId()));
waitFor(120 * 1000, new Predicate() {
@@ -593,6 +601,6 @@ public class TestMapReduceActionExecutor
// hadoop.counters will always be set in case of MR action.
assertNotNull(context.getVar("hadoop.counters"));
String counters = context.getVar("hadoop.counters");
- assertTrue(counters.contains("Task$Counter"));
+ assertTrue(counters.contains("Counter"));
}
-}
\ No newline at end of file
+}
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
Fri Feb 10 23:39:19 2012
@@ -44,6 +44,9 @@ public class TestMapReduceMain extends M
Path outputDir = new Path(getFsTestCaseDir(), "output");
XConfiguration jobConf = new XConfiguration();
+
+ jobConf.set("mapreduce.framework.name", "yarn");
+
jobConf.setInt("mapred.map.tasks", 1);
jobConf.setInt("mapred.map.max.attempts", 1);
jobConf.setInt("mapred.reduce.max.attempts", 1);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPipesMain.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPipesMain.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPipesMain.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestPipesMain.java
Fri Feb 10 23:39:19 2012
@@ -37,18 +37,8 @@ import java.net.URI;
public class TestPipesMain extends MainTestCase {
public static String getProgramName(XTestCase tc) {
- String hadoopVersion = tc.getHadoopVersion();
-
- String hadoopSuffix = null;
String osSuffix = null;
- if (hadoopVersion.startsWith("0.20")) {
- hadoopSuffix = "h20";
- }
- else {
- fail();
- }
-
if (System.getProperty("os.name").startsWith("Mac") &&
System.getProperty("os.arch").equals("x86_64")) {
osSuffix = "Mac_OS_X-x86_64-64";
}
@@ -62,7 +52,7 @@ public class TestPipesMain extends MainT
fail();
}
- return "wordcount-simple" + "_" + osSuffix + "_" + hadoopSuffix;
+ return "wordcount-simple" + "_" + osSuffix + "_h20";
}
public Void call() throws Exception {
@@ -95,6 +85,7 @@ public class TestPipesMain extends MainT
jobConf.set("mapred.job.tracker", getJobTrackerUri());
jobConf.set("fs.default.name", getNameNodeUri());
+ jobConf.set("mapreduce.framework.name", "yarn");
jobConf.set("mapred.input.dir", inputDir.toString());
jobConf.set("mapred.output.dir", outputDir.toString());
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
Fri Feb 10 23:39:19 2012
@@ -287,7 +287,8 @@ public class TestShellActionExecutor ext
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
injectKerberosInfo(conf);
- JobConf jobConf = new JobConf(conf);
+ JobConf jobConf = new JobConf();
+ XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
String group = jobConf.get("group.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, group,
jobConf);
Modified:
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
---
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
(original)
+++
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
Fri Feb 10 23:39:19 2012
@@ -59,6 +59,7 @@ public class TestStreamingMain extends M
jobConf.setInt("mapred.reduce.max.attempts", 1);
jobConf.set("mapred.job.tracker", getJobTrackerUri());
jobConf.set("fs.default.name", getNameNodeUri());
+ jobConf.set("mapreduce.framework.name", "yarn");
injectKerberosInfo(jobConf);
jobConf.set("user.name", getTestUser());
jobConf.set("hadoop.job.ugi", getTestUser() + "," + getTestGroup());
Modified: incubator/oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1242984&r1=1242983&r2=1242984&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Fri Feb 10 23:39:19 2012
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-691 Fix testcases using launcher using Hadoop 0.23.x (tucu)
OOZIE-690 use hadoop-client/hadoop-minicluster artifacts for Hadoop 0.23.1 &
trunk (tucu)
OOZIE-689 XTestCase proxyuser settings fails with Hadoop 1.0.0/0.23.1 (tucu)
OOZIE-582 Adding new test cases for the feature - viewing log for coordinator
actions in a given date range (harsh via tucu)
