I have bunch of jars whcih i want to pass. I am using libjars option to do
so. But to do that I have to implement tool ??
So i change my code to following but still I am getting this warning?
13/08/27 11:32:37 WARN mapred.JobClient: Use GenericOptionsParser for
parsing the arguments. Applications should implement Tool for the same.
and it is not finding that jar (in bold??)
package org.foobar;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
*import com.hp.hpl.jena.rdf.model.*;*
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class RdfFormatter extends Configured implements Tool {
//foo bar
}
public int run(String [] args) throws Exception{
//Configuration conf = new Configuration();
Job job = new Job();
//job.setMapOutputKeyClass(Pair.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
//conf.set("delimiter", "\\s+");
job.setJarByClass(RdfFormatter.class);
job.setMapperClass(RDFMapper.class);
//job.setReducerClass(MeanReducer.class);
job.setNumReduceTasks(0);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new
Path("/user/user/mohit/wiki-rdf/sample-input"));
FileOutputFormat.setOutputPath(job, new
Path("/user/user/mohit/wiki-rdf/outputter"));
job.submit();
long start = new Date().getTime();
job.waitForCompletion(true);
long end = new Date().getTime();
System.out.println("Job took " + ((end-start)/1000) + " seconds" );
return 0;
}
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
/*
Configuration conf = new Configuration();
Job job = new Job(conf, "RDF input format");
//job.setMapOutputKeyClass(Pair.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
conf.set("delimiter", "\\s+");
job.setJarByClass(Formatter.class);
job.setMapperClass(Mapper.class);
//job.setReducerClass(Reducer.class);
job.setNumReduceTasks(0);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path("/input/data"));
FileOutputFormat.setOutputPath(job, new Path("/output"));
// set timer
*
*/
Configuration conf = new Configuration();
int ret = ToolRunner.run(conf, new RdfFormatter(), args); // calls your
run() method.
System.exit(ret);
}
}