HBASE-11691 Backport HBASE-7156 (Add Data Block Encoding and -D opts to Performance Evaluation) to 0.94
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a8142d1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a8142d1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a8142d1 Branch: refs/heads/0.94 Commit: 9a8142d11f39f37a24486f77ac989a483643a1ba Parents: 46c2694 Author: Andrew Purtell <[email protected]> Authored: Thu Aug 7 15:22:42 2014 -0700 Committer: Andrew Purtell <[email protected]> Committed: Thu Aug 7 15:22:42 2014 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/PerformanceEvaluation.java | 56 +++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9a8142d1/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 37ad8e2..cca8210 100644 --- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -42,6 +42,7 @@ import java.lang.reflect.Constructor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.Compression; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.util.Bytes; @@ -82,6 +84,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer; import org.apache.hadoop.util.LineReader; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; /** * Script used evaluating HBase performance and scalability. Runs a HBase @@ -99,7 +103,7 @@ import org.apache.hadoop.util.LineReader; * <p>If number of clients > 1, we start up a MapReduce job. Each map task * runs an individual client. Each client does about 1GB of data. */ -public class PerformanceEvaluation { +public class PerformanceEvaluation extends Configured implements Tool { protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName()); public static final byte[] COMPRESSION = Bytes.toBytes("NONE"); @@ -119,13 +123,13 @@ public class PerformanceEvaluation { protected static HTableDescriptor TABLE_DESCRIPTOR; protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>(); - volatile Configuration conf; private boolean miniCluster = false; private boolean nomapred = false; private int N = 1; private int R = ROWS_PER_GB; private byte[] tableName = TABLE_NAME; private Compression.Algorithm compression = Compression.Algorithm.NONE; + private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE; private float sampleRate = 1.0f; private boolean flushCommits = true; private boolean reportLatency = false; @@ -158,10 +162,10 @@ public class PerformanceEvaluation { /** * Constructor - * @param c Configuration object + * @param conf Configuration object */ - public PerformanceEvaluation(final Configuration c) { - this.conf = c; + public PerformanceEvaluation(final Configuration conf) { + super(conf); addCommandDescriptor(RandomReadTest.class, "randomRead", "Run random read test"); @@ -517,6 +521,7 @@ public class PerformanceEvaluation { if (TABLE_DESCRIPTOR == null) { TABLE_DESCRIPTOR = new HTableDescriptor(tableName); HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME); + family.setDataBlockEncoding(blockEncoding); family.setCompressionType(compression); TABLE_DESCRIPTOR.addFamily(family); } @@ -549,7 +554,7 @@ public class PerformanceEvaluation { */ private void runNIsMoreThanOne(final Class<? extends Test> cmd) throws IOException, InterruptedException, ClassNotFoundException { - checkTable(new HBaseAdmin(conf)); + checkTable(new HBaseAdmin(getConf())); if (this.nomapred) { doMultipleClients(cmd); } else { @@ -570,7 +575,7 @@ public class PerformanceEvaluation { @Override public void run() { super.run(); - PerformanceEvaluation pe = new PerformanceEvaluation(conf); + PerformanceEvaluation pe = new PerformanceEvaluation(getConf()); int index = Integer.parseInt(getName()); try { long elapsedTime = pe.runOneClient(cmd, index * perClientRows, @@ -612,10 +617,11 @@ public class PerformanceEvaluation { */ private void doMapReduce(final Class<? extends Test> cmd) throws IOException, InterruptedException, ClassNotFoundException { - Path inputDir = writeInputFile(this.conf); - this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName()); - this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName()); - Job job = new Job(this.conf); + Configuration conf = getConf(); + Path inputDir = writeInputFile(conf); + conf.set(EvaluationMapTask.CMD_KEY, cmd.getName()); + conf.set(EvaluationMapTask.PE_KEY, getClass().getName()); + Job job = new Job(conf); job.setJarByClass(PerformanceEvaluation.class); job.setJobName("HBase Performance Evaluation"); @@ -1229,7 +1235,7 @@ public class PerformanceEvaluation { try { Constructor<? extends Test> constructor = cmd.getDeclaredConstructor( Configuration.class, TestOptions.class, Status.class); - t = constructor.newInstance(this.conf, options, status); + t = constructor.newInstance(getConf(), options, status); } catch (NoSuchMethodException e) { throw new IllegalArgumentException("Invalid command class: " + cmd.getName() + ". It does not provide a constructor as described by" + @@ -1255,7 +1261,7 @@ public class PerformanceEvaluation { HBaseAdmin admin = null; try { - admin = new HBaseAdmin(this.conf); + admin = new HBaseAdmin(getConf()); checkTable(admin); runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits, this.writeToWAL, this.writeToWAL, status); @@ -1271,6 +1277,7 @@ public class PerformanceEvaluation { MiniHBaseCluster hbaseMiniCluster = null; MiniDFSCluster dfsCluster = null; MiniZooKeeperCluster zooKeeperCluster = null; + Configuration conf = getConf(); if (this.miniCluster) { dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null); zooKeeperCluster = new MiniZooKeeperCluster(); @@ -1285,7 +1292,7 @@ public class PerformanceEvaluation { conf.set(HConstants.HBASE_DIR, parentdir.toString()); fs.mkdirs(parentdir); FSUtils.setVersion(fs, parentdir); - hbaseMiniCluster = new MiniHBaseCluster(this.conf, N); + hbaseMiniCluster = new MiniHBaseCluster(conf, N); } try { @@ -1316,7 +1323,7 @@ public class PerformanceEvaluation { } System.err.println("Usage: java " + this.getClass().getName() + " \\"); System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\"); - System.err.println(" [--compress=TYPE] <command> <nclients>"); + System.err.println(" [--compress=TYPE] [--blockEncoding=TYPE] [-D<property=value>]* <command> <nclients>"); System.err.println(); System.err.println("Options:"); System.err.println(" miniCluster Run the test on an HBaseMiniCluster"); @@ -1335,6 +1342,11 @@ public class PerformanceEvaluation { System.err.println(" latency Set to report operation latencies. " + "Currently only supported by randomRead test. Default: False"); System.err.println(); + System.err.println(" Note: -D properties will be applied to the conf used. "); + System.err.println(" For example: "); + System.err.println(" -Dmapred.output.compress=true"); + System.err.println(" -Dmapreduce.task.timeout=60000"); + System.err.println(); System.err.println("Command:"); for (CmdDescriptor command : commands.values()) { System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription())); @@ -1362,7 +1374,7 @@ public class PerformanceEvaluation { this.R = this.R * N; } - public int doCommandLine(final String[] args) { + public int run(String[] args) throws Exception { // Process command-line args. TODO: Better cmd-line processing // (but hopefully something not as painful as cli options). int errCode = -1; @@ -1416,6 +1428,12 @@ public class PerformanceEvaluation { continue; } + final String blockEncoding = "--blockEncoding="; + if (cmd.startsWith(blockEncoding)) { + this.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length())); + continue; + } + final String flushCommits = "--flushCommits="; if (cmd.startsWith(flushCommits)) { this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length())); @@ -1463,8 +1481,8 @@ public class PerformanceEvaluation { return descriptor != null ? descriptor.getCmdClass() : null; } - public static void main(final String[] args) { - Configuration c = HBaseConfiguration.create(); - System.exit(new PerformanceEvaluation(c).doCommandLine(args)); + public static void main(final String[] args) throws Exception { + int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args); + System.exit(res); } }
