Thanks for the quick response Mahesh. I am using the following command:
sudo -u hdfs hadoop jar word_cnt.jar WordCount2 /tmp/root/input /tmp/root/output15 -libjars=word_cnt.jar (The input directory exists on the hdfs) This is how I compiled and packaged it: javac -classpath /usr/lib/hadoop-0.20-mapreduce/hadoop-core.jar:/usr/lib/hadoop/* -d word_cnt WordCount2.java jar -cvf word_cnt.jar -C word_cnt/ . On Thu, Nov 29, 2012 at 10:46 AM, Mahesh Balija <[email protected]>wrote: > Hi Sandeep, > > For me everything seems to be alright. > Can you tell us how are you running this job? > > Best, > Mahesh.B. > Calsoft Labs. > > On Thu, Nov 29, 2012 at 9:01 PM, Sandeep Jangra > <[email protected]>wrote: > >> Hello everyone, >> >> Like most others I am also running into some problems while running my >> word count example. >> I tried the various suggestion available on internet, but I guess it;s >> time to go on email :) >> >> Here is the error that I am getting: >> 12/11/29 10:20:59 WARN mapred.JobClient: Use GenericOptionsParser for >> parsing the arguments. Applications should implement Tool for the same. >> 12/11/29 10:20:59 WARN mapred.JobClient: No job jar file set. User >> classes may not be found. See JobConf(Class) or JobConf#setJar(String). >> 12/11/29 10:20:59 INFO input.FileInputFormat: Total input paths to >> process : 1 >> 12/11/29 10:20:59 INFO util.NativeCodeLoader: Loaded the native-hadoop >> library >> 12/11/29 10:20:59 WARN snappy.LoadSnappy: Snappy native library is >> available >> 12/11/29 10:20:59 INFO snappy.LoadSnappy: Snappy native library loaded >> 12/11/29 10:21:00 INFO mapred.JobClient: Running job: >> job_201210310210_0040 >> 12/11/29 10:21:01 INFO mapred.JobClient: map 0% reduce 0% >> 12/11/29 10:21:07 INFO mapred.JobClient: Task Id : >> attempt_201210310210_0040_m_000000_0, Status : FAILED >> java.lang.RuntimeException: *java.lang.ClassNotFoundException: Class >> WordCount2$Map *not found >> at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1439) >> at >> org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:191) >> at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:605) >> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) >> at org.apache.hadoop.mapred.Child$4.run(Child.java:270) >> at java.security.AccessController.doPrivileged(Native Method) >> at javax.security.auth.Subject.doAs(Subject.java:416) >> at >> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1232) >> at org.apache.hadoop.mapred.Child.main(Child.java:264) >> Caused by: java.lang.ClassNotFoundException: Class WordCount2$Map not >> found >> at >> org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1350) >> at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1437) >> ... 8 more >> >> And here is the source code: >> >> >> import org.apache.hadoop.conf.Configuration; >> import org.apache.hadoop.conf.Configured; >> import org.apache.hadoop.fs.Path; >> import org.apache.hadoop.io.IntWritable; >> import org.apache.hadoop.io.LongWritable; >> import org.apache.hadoop.io.Text; >> import org.apache.hadoop.mapreduce.Job; >> import org.apache.hadoop.mapreduce.Mapper; >> import org.apache.hadoop.mapreduce.Reducer; >> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; >> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; >> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; >> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; >> import org.apache.hadoop.util.Tool; >> import org.apache.hadoop.util.ToolRunner; >> >> import java.io.IOException; >> import java.util.StringTokenizer; >> >> public class WordCount2 extends Configured implements Tool { >> >> >> public static class Map extends Mapper<LongWritable, Text, Text, >> IntWritable> { >> private final static IntWritable one = new IntWritable(1); >> private Text word = new Text(); >> >> @Override >> protected void map(LongWritable key, Text value, Context context) >> throws IOException, InterruptedException { >> String line = value.toString(); >> StringTokenizer tokenizer = new StringTokenizer(line); >> while (tokenizer.hasMoreTokens()) { >> word.set(tokenizer.nextToken()); >> context.write(word, one); >> } >> } >> >> } >> >> >> public static class Reduce extends Reducer<Text, IntWritable, Text, >> IntWritable> { >> >> @Override >> protected void reduce(Text key, Iterable<IntWritable> values, >> Context context) throws IOException, InterruptedException { >> >> int sum = 0; >> >> for(IntWritable value : values) { >> sum += value.get(); >> } >> // while (values.hasNext()) { >> // sum += values.next().get(); >> // } >> context.write(key, new IntWritable(sum)); >> } >> >> } >> >> @Override >> public int run(String[] args) throws Exception { >> Configuration conf = getConf(); >> for (java.util.Map.Entry<String, String> entry: conf) { >> System.out.printf("%s=%s\n", entry.getKey(), >> entry.getValue()); >> } >> >> System.out.println("arg[0]= "+args[0] + " args[1]= "+ args[1]); >> >> Job job = new Job(conf, WordCount2.class.getSimpleName()); >> job.setJobName("wordcount2"); >> job.setJarByClass(WordCount2.class); >> >> job.setMapOutputKeyClass(Text.class); >> job.setMapOutputValueClass(IntWritable.class); >> >> job.setOutputKeyClass(Text.class); >> job.setOutputValueClass(IntWritable.class); >> >> job.setMapperClass(Map.class); >> job.setCombinerClass(Reduce.class); >> job.setReducerClass(Reduce.class); >> >> job.setInputFormatClass(TextInputFormat.class); >> job.setOutputFormatClass(TextOutputFormat.class); >> >> FileInputFormat.setInputPaths(job, new Path(args[0])); >> FileOutputFormat.setOutputPath(job, new Path(args[1])); >> >> System.exit(job.waitForCompletion(true) ? 0 : 1); >> >> return 0; >> } >> >> >> >> public static void main(String[] args) throws Exception { >> int exitCode = ToolRunner.run(new WordCount2(), args); >> System.exit(exitCode); >> } >> } >> >> >> >
