Author: ddas
Date: Mon Nov 17 04:33:01 2008
New Revision: 718232
URL: http://svn.apache.org/viewvc?rev=718232&view=rev
Log:
HADOOP-4612. Removes RunJar's dependency on JobClient. Contributed by Sharad
Agarwal.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamInputFormat.java
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Nov 17 04:33:01 2008
@@ -108,6 +108,9 @@
HADOOP-4126. Allow access to HDFS web UI on EC2 (tomwhite via omalley)
+ HADOOP-4612. Removes RunJar's dependency on JobClient.
+ (Sharad Agarwal via ddas)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamInputFormat.java?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamInputFormat.java
(original)
+++
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamInputFormat.java
Mon Nov 17 04:33:01 2008
@@ -55,7 +55,7 @@
Class readerClass;
{
- readerClass = StreamUtil.goodClassOrNull(c, null);
+ readerClass = StreamUtil.goodClassOrNull(job, c, null);
if (readerClass == null) {
throw new RuntimeException("Class not found: " + c);
}
Modified:
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
Mon Nov 17 04:33:01 2008
@@ -705,7 +705,7 @@
||
inputFormatSpec_.equals(SequenceFileAsTextInputFormat.class.getSimpleName())) {
fmt = SequenceFileAsTextInputFormat.class;
} else {
- c = StreamUtil.goodClassOrNull(inputFormatSpec_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, inputFormatSpec_,
defaultPackage);
if (c != null) {
fmt = c;
} else {
@@ -725,7 +725,7 @@
jobConf_.set("stream.addenvironment", addTaskEnvironment_);
if (mapCmd_ != null) {
- c = StreamUtil.goodClassOrNull(mapCmd_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, mapCmd_, defaultPackage);
if (c != null) {
jobConf_.setMapperClass(c);
} else {
@@ -736,7 +736,7 @@
}
if (comCmd_ != null) {
- c = StreamUtil.goodClassOrNull(comCmd_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, comCmd_, defaultPackage);
if (c != null) {
jobConf_.setCombinerClass(c);
} else {
@@ -752,7 +752,7 @@
jobConf_.setCombinerClass(ValueAggregatorCombiner.class);
} else {
- c = StreamUtil.goodClassOrNull(redCmd_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, redCmd_, defaultPackage);
if (c != null) {
jobConf_.setReducerClass(c);
} else {
@@ -767,7 +767,7 @@
String[] args = inReaderSpec_.split(",");
String readerClass = args[0];
// this argument can only be a Java class
- c = StreamUtil.goodClassOrNull(readerClass, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, readerClass, defaultPackage);
if (c != null) {
jobConf_.set("stream.recordreader.class", c.getName());
} else {
@@ -784,7 +784,7 @@
FileOutputFormat.setOutputPath(jobConf_, new Path(output_));
fmt = null;
if (outputFormatSpec_!= null) {
- c = StreamUtil.goodClassOrNull(outputFormatSpec_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, outputFormatSpec_,
defaultPackage);
if (c != null) {
fmt = c;
} else {
@@ -797,7 +797,7 @@
jobConf_.setOutputFormat(fmt);
if (partitionerSpec_!= null) {
- c = StreamUtil.goodClassOrNull(partitionerSpec_, defaultPackage);
+ c = StreamUtil.goodClassOrNull(jobConf_, partitionerSpec_,
defaultPackage);
if (c != null) {
jobConf_.setPartitionerClass(c);
} else {
Modified:
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
---
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
(original)
+++
hadoop/core/trunk/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamUtil.java
Mon Nov 17 04:33:01 2008
@@ -28,6 +28,7 @@
import java.util.List;
import java.util.jar.*;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -46,15 +47,14 @@
* [-mapper program | -javamapper classname], -mapper and -javamapper are
mutually exclusive.
* (repeat for -reducer, -combiner) </pre>
*/
- public static Class goodClassOrNull(String className, String defaultPackage)
{
+ public static Class goodClassOrNull(Configuration conf, String className,
String defaultPackage) {
if (className.indexOf('.') == -1 && defaultPackage != null) {
className = defaultPackage + "." + className;
}
Class clazz = null;
try {
- clazz = Class.forName(className);
+ clazz = conf.getClassByName(className);
} catch (ClassNotFoundException cnf) {
- } catch (LinkageError cnf) {
}
return clazz;
}
Modified:
hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/GenericOptionsParser.java
Mon Nov 17 04:33:01 2008
@@ -234,10 +234,13 @@
conf.set("tmpjars",
validateFiles(line.getOptionValue("libjars"), conf));
//setting libjars in client classpath
- ClassLoader loader = new URLClassLoader(getLibJars(conf),
- conf.getClassLoader());
- Thread.currentThread().setContextClassLoader(loader);
- conf.setClassLoader(loader);
+ URL[] libjars = getLibJars(conf);
+ if(libjars!=null && libjars.length>0) {
+ conf.setClassLoader(new URLClassLoader(libjars,
conf.getClassLoader()));
+ Thread.currentThread().setContextClassLoader(
+ new URLClassLoader(libjars,
+ Thread.currentThread().getContextClassLoader()));
+ }
}
if (line.hasOption("files")) {
conf.set("tmpfiles",
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java?rev=718232&r1=718231&r2=718232&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/RunJar.java Mon Nov 17
04:33:01 2008
@@ -27,7 +27,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.mapred.JobClient;
/** Run a Hadoop job jar. */
public class RunJar {
@@ -143,14 +142,6 @@
}
}
- //adding libjars to the classpath
- Configuration conf = JobClient.getCommandLineConfig();
- URL[] libJars = GenericOptionsParser.getLibJars(conf);
- if(libJars!=null) {
- for(URL url : libJars){
- classPath.add(url);
- }
- }
ClassLoader loader =
new URLClassLoader(classPath.toArray(new URL[0]));
