Steven, I would recommend putting 1 region server with each datanode and use 3 or 5 quorum servers, 4 means that your quorum is already crippled of one member.
I would also recommend reviewing the following doc: http://hadoop.apache.org/hbase/docs/r0.20.3/api/org/apache/hadoop/hbase/mapreduce/package-summary.html See how the classpath is handled and try to do the same. It's how I personally do it. J-D On Wed, Feb 24, 2010 at 6:46 PM, steven zhuang <zhuangxin8...@gmail.com> wrote: > hello, > I have a program loading formated data into HTable, but it crashed > when connecting to the quorum server because of wrong configuration, seems > that it load configuration from hbase-default.xml, not from hbase-site.xml. > > my hadoop cluster has 35 nodes, and the administrator has set up 4 > quorum server and 5 region server(too few?). we have copied the Hbase > configuration folder to every hadoop task tracker. > > the job failed during the reduce phase, raised exceptions " > NoServerForRegionException", I have checked the job log on some nodes, the > connection string is set to "localhost:port" instead of the configuration in > hbase-site.xml. > > my code is as follows, I think the problem may be from the *doJob > method(when I do the configuration):* > please help. > * > * > *import java.io.IOException;* > *import java.util.Iterator;* > * > * > *import org.apache.hadoop.conf.Configuration;* > *import org.apache.hadoop.conf.Configured;* > *import org.apache.hadoop.hbase.HBaseConfiguration;* > *import org.apache.hadoop.hbase.HColumnDescriptor;* > *import org.apache.hadoop.hbase.HTableDescriptor;* > *import org.apache.hadoop.hbase.TableExistsException;* > *import org.apache.hadoop.hbase.client.HBaseAdmin;* > *import org.apache.hadoop.hbase.client.Put;* > *import org.apache.hadoop.hbase.io.HbaseMapWritable;* > *import org.apache.hadoop.hbase.io.ImmutableBytesWritable;* > *import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;* > *import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;* > *import org.apache.hadoop.hbase.mapreduce.TableReducer;* > *import org.apache.hadoop.hbase.util.Bytes;* > *import org.apache.hadoop.io.LongWritable;* > *import org.apache.hadoop.io.Text;* > *import org.apache.hadoop.mapred.JobClient;* > *import org.apache.hadoop.mapred.JobConf;* > *import org.apache.hadoop.mapreduce.Job;* > *import org.apache.hadoop.mapreduce.Mapper;* > *import org.apache.hadoop.mapreduce.Mapper.Context;* > *import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;* > *import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;* > *import org.apache.hadoop.util.Tool;* > * > * > *public class FasterMidImport extends Configured {* > * > * > *......* > * **private static Configuration conf = new HBaseConfiguration();* > * > * > * **public static class InnerMap extends Mapper<LongWritable, Text, Text, > Text> {* > * > * > * **protected void map(LongWritable key, Text value, Context context)* > * **throws IOException, InterruptedException {* > * ......* > * **context.write(new Text(rowKey), new Text(columnid + "\t" + value));* > * **}* > * **}* > * > * > * **public static class MyTableReducer extends TableReducer<Text, Text, > Text> {* > * **protected void reduce(Text key, Iterable<Text> values, Context context)* > * **throws IOException, InterruptedException {* > * **Iterator<Text> iter = values.iterator();* > * **Put put = new Put(key.toString().getBytes());* > * **int cellCount = 0;* > * **while (iter.hasNext()) {* > * **String[] splits = iter.next().toString().split("\t");* > * **if (splits.length < 2) {* > * **continue;* > * **}* > * **String query = splits[0].substring(splits[0].indexOf(':') + 1);* > * **put.add("queries".getBytes(), query.getBytes(), splits[1]* > * **.getBytes());* > * **if (++cellCount > 10000) {* > * **cellCount = 0;* > * **context.write(key, put);* > * **put = new Put(key.toString().getBytes());* > * **}* > * **}* > * **if (0 < cellCount) {* > * **context.write(key, put);* > * **}* > * **}* > * **}* > * > * > * **public void doJob(String[] args) throws IOException {* > * **conf.set("input.dir", args[0]);* > * **conf.set("table.name", args[1]);* > * **try {* > * **String tblName = conf.get("table.name");* > * **try {* > * **HBaseAdmin admin = new HBaseAdmin((HBaseConfiguration) conf);* > * **if (!admin.tableExists(tblName)) {* > * **Thread.currentThread().sleep(2000);* > * **HTableDescriptor tblDesc = new HTableDescriptor(tblName);* > * **tblDesc.addFamily(new HColumnDescriptor("queries"));* > * **admin.createTable(tblDesc);* > * **}* > * **} catch (TableExistsException e) {* > * **} catch (InterruptedException e) {* > * **e.printStackTrace();* > * **}* > * **} catch (IOException e) {* > * **e.printStackTrace();* > * **}* > * **Job job = new Job(conf);* > * **// Job job = new Job(new Configuration());* > * **job.setJarByClass(FasterMidImport.class);* > * **job.setJobName("Faster-midImport: import " + conf.get("input.dir")* > * **+ "into" + conf.get("input.dir"));* > * > * > * **job.setOutputKeyClass(Text.class);* > * **job.setOutputValueClass(Text.class);* > * **job.setOutputFormatClass(TableOutputFormat.class);* > * > * > * **job.setInputFormatClass(TextInputFormat.class);* > * **FileInputFormat.setInputPaths(job, conf.get("input.dir"));* > * > * > * **job.setMapperClass(InnerMap.class);* > * > * > * **job.setMapOutputKeyClass(Text.class);* > * **job.setMapOutputValueClass(Text.class);* > * > * > * **job.setNumReduceTasks(20);* > * **// > ////////////////////////////////////////////////////////////////////////////////////////////////////////////// > * > * **TableMapReduceUtil.initTableReducerJob(conf.get("table.name"),* > * **MyTableReducer.class, job);* > * **try {* > * **job.submit();* > * **while (!job.isComplete()) {* > * **Thread.currentThread().sleep(1000);* > * **System.out* > * **.println("Map: " + (job.mapProgress() * 100)* > * **+ "% ... Reduce: "* > * **+ (job.reduceProgress() * 100) + "%");* > * > * > * **}* > * **} catch (InterruptedException e) {* > * **e.printStackTrace();* > * **} catch (ClassNotFoundException e) {* > * **e.printStackTrace();* > * **}* > * **}* > * **static int printUsage() {* > * **System.err.println("Usage: FasterMidImport <input-dir> <table_name>");* > * **return -1;* > * **}* > * **/*** > * ** * * Run* > * ** * */* > * **public int run(String[] args) throws Exception {* > * **if (args.length < 2) {* > * **return printUsage();* > * **}* > * **this.doJob(args);* > * **return 0;* > * **}* > * > * > * **public static int main(String[] args) throws Exception {* > * **new FasterMidImport().run(args);* > * **return 0;* > * **}* > *}* > > > -- > best wishes. > steven >
