hello,
I have a program loading formated data into HTable, but it crashed
when connecting to the quorum server because of wrong configuration, seems
that it load configuration from hbase-default.xml, not from hbase-site.xml.
my hadoop cluster has 35 nodes, and the administrator has set up 4
quorum server and 5 region server(too few?). we have copied the Hbase
configuration folder to every hadoop task tracker.
the job failed during the reduce phase, raised exceptions "
NoServerForRegionException", I have checked the job log on some nodes, the
connection string is set to "localhost:port" instead of the configuration in
hbase-site.xml.
my code is as follows, I think the problem may be from the *doJob
method(when I do the configuration):*
please help.
*
*
*import java.io.IOException;*
*import java.util.Iterator;*
*
*
*import org.apache.hadoop.conf.Configuration;*
*import org.apache.hadoop.conf.Configured;*
*import org.apache.hadoop.hbase.HBaseConfiguration;*
*import org.apache.hadoop.hbase.HColumnDescriptor;*
*import org.apache.hadoop.hbase.HTableDescriptor;*
*import org.apache.hadoop.hbase.TableExistsException;*
*import org.apache.hadoop.hbase.client.HBaseAdmin;*
*import org.apache.hadoop.hbase.client.Put;*
*import org.apache.hadoop.hbase.io.HbaseMapWritable;*
*import org.apache.hadoop.hbase.io.ImmutableBytesWritable;*
*import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;*
*import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;*
*import org.apache.hadoop.hbase.mapreduce.TableReducer;*
*import org.apache.hadoop.hbase.util.Bytes;*
*import org.apache.hadoop.io.LongWritable;*
*import org.apache.hadoop.io.Text;*
*import org.apache.hadoop.mapred.JobClient;*
*import org.apache.hadoop.mapred.JobConf;*
*import org.apache.hadoop.mapreduce.Job;*
*import org.apache.hadoop.mapreduce.Mapper;*
*import org.apache.hadoop.mapreduce.Mapper.Context;*
*import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;*
*import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;*
*import org.apache.hadoop.util.Tool;*
*
*
*public class FasterMidImport extends Configured {*
*
*
*......*
* **private static Configuration conf = new HBaseConfiguration();*
*
*
* **public static class InnerMap extends Mapper<LongWritable, Text, Text,
Text> {*
*
*
* **protected void map(LongWritable key, Text value, Context context)*
* **throws IOException, InterruptedException {*
* ......*
* **context.write(new Text(rowKey), new Text(columnid + "\t" + value));*
* **}*
* **}*
*
*
* **public static class MyTableReducer extends TableReducer<Text, Text,
Text> {*
* **protected void reduce(Text key, Iterable<Text> values, Context context)*
* **throws IOException, InterruptedException {*
* **Iterator<Text> iter = values.iterator();*
* **Put put = new Put(key.toString().getBytes());*
* **int cellCount = 0;*
* **while (iter.hasNext()) {*
* **String[] splits = iter.next().toString().split("\t");*
* **if (splits.length < 2) {*
* **continue;*
* **}*
* **String query = splits[0].substring(splits[0].indexOf(':') + 1);*
* **put.add("queries".getBytes(), query.getBytes(), splits[1]*
* **.getBytes());*
* **if (++cellCount > 10000) {*
* **cellCount = 0;*
* **context.write(key, put);*
* **put = new Put(key.toString().getBytes());*
* **}*
* **}*
* **if (0 < cellCount) {*
* **context.write(key, put);*
* **}*
* **}*
* **}*
*
*
* **public void doJob(String[] args) throws IOException {*
* **conf.set("input.dir", args[0]);*
* **conf.set("table.name", args[1]);*
* **try {*
* **String tblName = conf.get("table.name");*
* **try {*
* **HBaseAdmin admin = new HBaseAdmin((HBaseConfiguration) conf);*
* **if (!admin.tableExists(tblName)) {*
* **Thread.currentThread().sleep(2000);*
* **HTableDescriptor tblDesc = new HTableDescriptor(tblName);*
* **tblDesc.addFamily(new HColumnDescriptor("queries"));*
* **admin.createTable(tblDesc);*
* **}*
* **} catch (TableExistsException e) {*
* **} catch (InterruptedException e) {*
* **e.printStackTrace();*
* **}*
* **} catch (IOException e) {*
* **e.printStackTrace();*
* **}*
* **Job job = new Job(conf);*
* **// Job job = new Job(new Configuration());*
* **job.setJarByClass(FasterMidImport.class);*
* **job.setJobName("Faster-midImport: import " + conf.get("input.dir")*
* **+ "into" + conf.get("input.dir"));*
*
*
* **job.setOutputKeyClass(Text.class);*
* **job.setOutputValueClass(Text.class);*
* **job.setOutputFormatClass(TableOutputFormat.class);*
*
*
* **job.setInputFormatClass(TextInputFormat.class);*
* **FileInputFormat.setInputPaths(job, conf.get("input.dir"));*
*
*
* **job.setMapperClass(InnerMap.class);*
*
*
* **job.setMapOutputKeyClass(Text.class);*
* **job.setMapOutputValueClass(Text.class);*
*
*
* **job.setNumReduceTasks(20);*
* **//
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
*
* **TableMapReduceUtil.initTableReducerJob(conf.get("table.name"),*
* **MyTableReducer.class, job);*
* **try {*
* **job.submit();*
* **while (!job.isComplete()) {*
* **Thread.currentThread().sleep(1000);*
* **System.out*
* **.println("Map: " + (job.mapProgress() * 100)*
* **+ "% ... Reduce: "*
* **+ (job.reduceProgress() * 100) + "%");*
*
*
* **}*
* **} catch (InterruptedException e) {*
* **e.printStackTrace();*
* **} catch (ClassNotFoundException e) {*
* **e.printStackTrace();*
* **}*
* **}*
* **static int printUsage() {*
* **System.err.println("Usage: FasterMidImport <input-dir> <table_name>");*
* **return -1;*
* **}*
* **/***
* ** * * Run*
* ** * */*
* **public int run(String[] args) throws Exception {*
* **if (args.length < 2) {*
* **return printUsage();*
* **}*
* **this.doJob(args);*
* **return 0;*
* **}*
*
*
* **public static int main(String[] args) throws Exception {*
* **new FasterMidImport().run(args);*
* **return 0;*
* **}*
*}*
--
best wishes.
steven
