Repository: hbase Updated Branches: refs/heads/branch-2 95bc46452 -> 59d034106
http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java deleted file mode 100644 index ad832e3..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/LoadTestTool.java +++ /dev/null @@ -1,968 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.util; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.lang.reflect.Constructor; -import java.net.InetAddress; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.atomic.AtomicReference; - -import javax.crypto.spec.SecretKeySpec; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseInterfaceAudience; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.io.compress.Compression; -import org.apache.hadoop.hbase.io.crypto.Cipher; -import org.apache.hadoop.hbase.io.crypto.Encryption; -import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.regionserver.BloomType; -import org.apache.hadoop.hbase.security.EncryptionUtil; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.access.AccessControlClient; -import org.apache.hadoop.hbase.security.access.Permission; -import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; -import org.apache.hadoop.hbase.util.test.LoadTestDataGeneratorWithACL; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ToolRunner; - -/** - * A command-line utility that reads, writes, and verifies data. Unlike - * {@link org.apache.hadoop.hbase.PerformanceEvaluation}, this tool validates the data written, - * and supports simultaneously writing and reading the same set of keys. - */ -@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS) -public class LoadTestTool extends AbstractHBaseTool { - - private static final Log LOG = LogFactory.getLog(LoadTestTool.class); - private static final String COLON = ":"; - - /** Table name for the test */ - private TableName tableName; - - /** Column families for the test */ - private byte[][] families; - - /** Table name to use of not overridden on the command line */ - protected static final String DEFAULT_TABLE_NAME = "cluster_test"; - - /** Column family used by the test */ - public static byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("test_cf"); - - /** Column families used by the test */ - public static final byte[][] DEFAULT_COLUMN_FAMILIES = { DEFAULT_COLUMN_FAMILY }; - - /** The default data size if not specified */ - protected static final int DEFAULT_DATA_SIZE = 64; - - /** The number of reader/writer threads if not specified */ - protected static final int DEFAULT_NUM_THREADS = 20; - - /** Usage string for the load option */ - protected static final String OPT_USAGE_LOAD = - "<avg_cols_per_key>:<avg_data_size>" + - "[:<#threads=" + DEFAULT_NUM_THREADS + ">]"; - - /** Usage string for the read option */ - protected static final String OPT_USAGE_READ = - "<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]"; - - /** Usage string for the update option */ - protected static final String OPT_USAGE_UPDATE = - "<update_percent>[:<#threads=" + DEFAULT_NUM_THREADS - + ">][:<#whether to ignore nonce collisions=0>]"; - - protected static final String OPT_USAGE_BLOOM = "Bloom filter type, one of " + - Arrays.toString(BloomType.values()); - - protected static final String OPT_USAGE_COMPRESSION = "Compression type, " + - "one of " + Arrays.toString(Compression.Algorithm.values()); - - public static final String OPT_DATA_BLOCK_ENCODING_USAGE = - "Encoding algorithm (e.g. prefix " - + "compression) to use for data blocks in the test column family, " - + "one of " + Arrays.toString(DataBlockEncoding.values()) + "."; - - public static final String OPT_BLOOM = "bloom"; - public static final String OPT_COMPRESSION = "compression"; - public static final String OPT_DEFERRED_LOG_FLUSH = "deferredlogflush"; - public static final String OPT_DEFERRED_LOG_FLUSH_USAGE = "Enable deferred log flush."; - - public static final String OPT_DATA_BLOCK_ENCODING = - HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase(Locale.ROOT); - - public static final String OPT_INMEMORY = "in_memory"; - public static final String OPT_USAGE_IN_MEMORY = "Tries to keep the HFiles of the CF " + - "inmemory as far as possible. Not guaranteed that reads are always served from inmemory"; - - public static final String OPT_GENERATOR = "generator"; - public static final String OPT_GENERATOR_USAGE = "The class which generates load for the tool." - + " Any args for this class can be passed as colon separated after class name"; - - public static final String OPT_WRITER = "writer"; - public static final String OPT_WRITER_USAGE = "The class for executing the write requests"; - - public static final String OPT_UPDATER = "updater"; - public static final String OPT_UPDATER_USAGE = "The class for executing the update requests"; - - public static final String OPT_READER = "reader"; - public static final String OPT_READER_USAGE = "The class for executing the read requests"; - - protected static final String OPT_KEY_WINDOW = "key_window"; - protected static final String OPT_WRITE = "write"; - protected static final String OPT_MAX_READ_ERRORS = "max_read_errors"; - public static final String OPT_MULTIPUT = "multiput"; - public static final String OPT_MULTIGET = "multiget_batchsize"; - protected static final String OPT_NUM_KEYS = "num_keys"; - protected static final String OPT_READ = "read"; - protected static final String OPT_START_KEY = "start_key"; - public static final String OPT_TABLE_NAME = "tn"; - public static final String OPT_COLUMN_FAMILIES = "families"; - protected static final String OPT_ZK_QUORUM = "zk"; - protected static final String OPT_ZK_PARENT_NODE = "zk_root"; - protected static final String OPT_SKIP_INIT = "skip_init"; - protected static final String OPT_INIT_ONLY = "init_only"; - protected static final String NUM_TABLES = "num_tables"; - protected static final String OPT_REGIONS_PER_SERVER = "regions_per_server"; - protected static final String OPT_BATCHUPDATE = "batchupdate"; - protected static final String OPT_UPDATE = "update"; - - public static final String OPT_ENCRYPTION = "encryption"; - protected static final String OPT_ENCRYPTION_USAGE = - "Enables transparent encryption on the test table, one of " + - Arrays.toString(Encryption.getSupportedCiphers()); - - public static final String OPT_NUM_REGIONS_PER_SERVER = "num_regions_per_server"; - protected static final String OPT_NUM_REGIONS_PER_SERVER_USAGE - = "Desired number of regions per region server. Defaults to 5."; - public static int DEFAULT_NUM_REGIONS_PER_SERVER = 5; - - public static final String OPT_REGION_REPLICATION = "region_replication"; - protected static final String OPT_REGION_REPLICATION_USAGE = - "Desired number of replicas per region"; - - public static final String OPT_REGION_REPLICA_ID = "region_replica_id"; - protected static final String OPT_REGION_REPLICA_ID_USAGE = - "Region replica id to do the reads from"; - - public static final String OPT_MOB_THRESHOLD = "mob_threshold"; - protected static final String OPT_MOB_THRESHOLD_USAGE = - "Desired cell size to exceed in bytes that will use the MOB write path"; - - protected static final long DEFAULT_START_KEY = 0; - - /** This will be removed as we factor out the dependency on command line */ - protected CommandLine cmd; - - protected MultiThreadedWriter writerThreads = null; - protected MultiThreadedReader readerThreads = null; - protected MultiThreadedUpdater updaterThreads = null; - - protected long startKey, endKey; - - protected boolean isWrite, isRead, isUpdate; - protected boolean deferredLogFlush; - - // Column family options - protected DataBlockEncoding dataBlockEncodingAlgo; - protected Compression.Algorithm compressAlgo; - protected BloomType bloomType; - private boolean inMemoryCF; - - private User userOwner; - // Writer options - protected int numWriterThreads = DEFAULT_NUM_THREADS; - protected int minColsPerKey, maxColsPerKey; - protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE; - protected boolean isMultiPut; - - // Updater options - protected int numUpdaterThreads = DEFAULT_NUM_THREADS; - protected int updatePercent; - protected boolean ignoreConflicts = false; - protected boolean isBatchUpdate; - - // Reader options - private int numReaderThreads = DEFAULT_NUM_THREADS; - private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW; - private int multiGetBatchSize = MultiThreadedReader.DEFAULT_BATCH_SIZE; - private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS; - private int verifyPercent; - - private int numTables = 1; - - private String superUser; - - private String userNames; - //This file is used to read authentication information in secure clusters. - private String authnFileName; - - private int numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER; - private int regionReplication = -1; // not set - private int regionReplicaId = -1; // not set - - private int mobThreshold = -1; // not set - - // TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad, - // console tool itself should only be used from console. - protected boolean isSkipInit = false; - protected boolean isInitOnly = false; - - protected Cipher cipher = null; - - protected String[] splitColonSeparated(String option, - int minNumCols, int maxNumCols) { - String optVal = cmd.getOptionValue(option); - String[] cols = optVal.split(COLON); - if (cols.length < minNumCols || cols.length > maxNumCols) { - throw new IllegalArgumentException("Expected at least " - + minNumCols + " columns but no more than " + maxNumCols + - " in the colon-separated value '" + optVal + "' of the " + - "-" + option + " option"); - } - return cols; - } - - protected int getNumThreads(String numThreadsStr) { - return parseInt(numThreadsStr, 1, Short.MAX_VALUE); - } - - public byte[][] getColumnFamilies() { - return families; - } - - /** - * Apply column family options such as Bloom filters, compression, and data - * block encoding. - */ - protected void applyColumnFamilyOptions(TableName tableName, - byte[][] columnFamilies) throws IOException { - try (Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin()) { - TableDescriptor tableDesc = admin.getTableDescriptor(tableName); - LOG.info("Disabling table " + tableName); - admin.disableTable(tableName); - for (byte[] cf : columnFamilies) { - HColumnDescriptor columnDesc = (HColumnDescriptor) tableDesc.getColumnFamily(cf); - boolean isNewCf = columnDesc == null; - if (isNewCf) { - columnDesc = new HColumnDescriptor(cf); - } - if (bloomType != null) { - columnDesc.setBloomFilterType(bloomType); - } - if (compressAlgo != null) { - columnDesc.setCompressionType(compressAlgo); - } - if (dataBlockEncodingAlgo != null) { - columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); - } - if (inMemoryCF) { - columnDesc.setInMemory(inMemoryCF); - } - if (cipher != null) { - byte[] keyBytes = new byte[cipher.getKeyLength()]; - new SecureRandom().nextBytes(keyBytes); - columnDesc.setEncryptionType(cipher.getName()); - columnDesc.setEncryptionKey( - EncryptionUtil.wrapKey(conf, - User.getCurrent().getShortName(), - new SecretKeySpec(keyBytes, - cipher.getName()))); - } - if (mobThreshold >= 0) { - columnDesc.setMobEnabled(true); - columnDesc.setMobThreshold(mobThreshold); - } - - if (isNewCf) { - admin.addColumnFamily(tableName, columnDesc); - } else { - admin.modifyColumnFamily(tableName, columnDesc); - } - } - LOG.info("Enabling table " + tableName); - admin.enableTable(tableName); - } - } - - @Override - protected void addOptions() { - addOptWithArg(OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " + - "without port numbers"); - addOptWithArg(OPT_ZK_PARENT_NODE, "name of parent znode in zookeeper"); - addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write"); - addOptWithArg(OPT_COLUMN_FAMILIES, "The name of the column families to use separated by comma"); - addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD); - addOptWithArg(OPT_READ, OPT_USAGE_READ); - addOptWithArg(OPT_UPDATE, OPT_USAGE_UPDATE); - addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading"); - addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM); - addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION); - addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE); - addOptWithArg(OPT_MAX_READ_ERRORS, "The maximum number of read errors " + - "to tolerate before terminating all reader threads. The default is " + - MultiThreadedReader.DEFAULT_MAX_ERRORS + "."); - addOptWithArg(OPT_MULTIGET, "Whether to use multi-gets as opposed to " + - "separate gets for every column in a row"); - addOptWithArg(OPT_KEY_WINDOW, "The 'key window' to maintain between " + - "reads and writes for concurrent write/read workload. The default " + - "is " + MultiThreadedReader.DEFAULT_KEY_WINDOW + "."); - - addOptNoArg(OPT_MULTIPUT, "Whether to use multi-puts as opposed to " + - "separate puts for every column in a row"); - addOptNoArg(OPT_BATCHUPDATE, "Whether to use batch as opposed to " + - "separate updates for every column in a row"); - addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY); - addOptWithArg(OPT_GENERATOR, OPT_GENERATOR_USAGE); - addOptWithArg(OPT_WRITER, OPT_WRITER_USAGE); - addOptWithArg(OPT_UPDATER, OPT_UPDATER_USAGE); - addOptWithArg(OPT_READER, OPT_READER_USAGE); - - addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write"); - addOptWithArg(OPT_START_KEY, "The first key to read/write " + - "(a 0-based index). The default value is " + - DEFAULT_START_KEY + "."); - addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table " - + "already exists"); - - addOptWithArg(NUM_TABLES, - "A positive integer number. When a number n is speicfied, load test " - + "tool will load n table parallely. -tn parameter value becomes " - + "table name prefix. Each table name is in format <tn>_1...<tn>_n"); - - addOptWithArg(OPT_REGIONS_PER_SERVER, - "A positive integer number. When a number n is specified, load test " - + "tool will create the test table with n regions per server"); - - addOptWithArg(OPT_ENCRYPTION, OPT_ENCRYPTION_USAGE); - addOptNoArg(OPT_DEFERRED_LOG_FLUSH, OPT_DEFERRED_LOG_FLUSH_USAGE); - addOptWithArg(OPT_NUM_REGIONS_PER_SERVER, OPT_NUM_REGIONS_PER_SERVER_USAGE); - addOptWithArg(OPT_REGION_REPLICATION, OPT_REGION_REPLICATION_USAGE); - addOptWithArg(OPT_REGION_REPLICA_ID, OPT_REGION_REPLICA_ID_USAGE); - addOptWithArg(OPT_MOB_THRESHOLD, OPT_MOB_THRESHOLD_USAGE); - } - - @Override - protected void processOptions(CommandLine cmd) { - this.cmd = cmd; - - tableName = TableName.valueOf(cmd.getOptionValue(OPT_TABLE_NAME, - DEFAULT_TABLE_NAME)); - - if (cmd.hasOption(OPT_COLUMN_FAMILIES)) { - String[] list = cmd.getOptionValue(OPT_COLUMN_FAMILIES).split(","); - families = new byte[list.length][]; - for (int i = 0; i < list.length; i++) { - families[i] = Bytes.toBytes(list[i]); - } - } else { - families = DEFAULT_COLUMN_FAMILIES; - } - - isWrite = cmd.hasOption(OPT_WRITE); - isRead = cmd.hasOption(OPT_READ); - isUpdate = cmd.hasOption(OPT_UPDATE); - isInitOnly = cmd.hasOption(OPT_INIT_ONLY); - deferredLogFlush = cmd.hasOption(OPT_DEFERRED_LOG_FLUSH); - - if (!isWrite && !isRead && !isUpdate && !isInitOnly) { - throw new IllegalArgumentException("Either -" + OPT_WRITE + " or " + - "-" + OPT_UPDATE + " or -" + OPT_READ + " has to be specified"); - } - - if (isInitOnly && (isRead || isWrite || isUpdate)) { - throw new IllegalArgumentException(OPT_INIT_ONLY + " cannot be specified with" - + " either -" + OPT_WRITE + " or -" + OPT_UPDATE + " or -" + OPT_READ); - } - - if (!isInitOnly) { - if (!cmd.hasOption(OPT_NUM_KEYS)) { - throw new IllegalArgumentException(OPT_NUM_KEYS + " must be specified in " - + "read or write mode"); - } - startKey = parseLong(cmd.getOptionValue(OPT_START_KEY, - String.valueOf(DEFAULT_START_KEY)), 0, Long.MAX_VALUE); - long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1, - Long.MAX_VALUE - startKey); - endKey = startKey + numKeys; - isSkipInit = cmd.hasOption(OPT_SKIP_INIT); - System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]"); - } - - parseColumnFamilyOptions(cmd); - - if (isWrite) { - String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3); - - int colIndex = 0; - minColsPerKey = 1; - maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]); - int avgColDataSize = - parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE); - minColDataSize = avgColDataSize / 2; - maxColDataSize = avgColDataSize * 3 / 2; - - if (colIndex < writeOpts.length) { - numWriterThreads = getNumThreads(writeOpts[colIndex++]); - } - - isMultiPut = cmd.hasOption(OPT_MULTIPUT); - - mobThreshold = -1; - if (cmd.hasOption(OPT_MOB_THRESHOLD)) { - mobThreshold = Integer.parseInt(cmd.getOptionValue(OPT_MOB_THRESHOLD)); - } - - System.out.println("Multi-puts: " + isMultiPut); - System.out.println("Columns per key: " + minColsPerKey + ".." - + maxColsPerKey); - System.out.println("Data size per column: " + minColDataSize + ".." - + maxColDataSize); - } - - if (isUpdate) { - String[] mutateOpts = splitColonSeparated(OPT_UPDATE, 1, 3); - int colIndex = 0; - updatePercent = parseInt(mutateOpts[colIndex++], 0, 100); - if (colIndex < mutateOpts.length) { - numUpdaterThreads = getNumThreads(mutateOpts[colIndex++]); - } - if (colIndex < mutateOpts.length) { - ignoreConflicts = parseInt(mutateOpts[colIndex++], 0, 1) == 1; - } - - isBatchUpdate = cmd.hasOption(OPT_BATCHUPDATE); - - System.out.println("Batch updates: " + isBatchUpdate); - System.out.println("Percent of keys to update: " + updatePercent); - System.out.println("Updater threads: " + numUpdaterThreads); - System.out.println("Ignore nonce conflicts: " + ignoreConflicts); - } - - if (isRead) { - String[] readOpts = splitColonSeparated(OPT_READ, 1, 2); - int colIndex = 0; - verifyPercent = parseInt(readOpts[colIndex++], 0, 100); - if (colIndex < readOpts.length) { - numReaderThreads = getNumThreads(readOpts[colIndex++]); - } - - if (cmd.hasOption(OPT_MAX_READ_ERRORS)) { - maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS), - 0, Integer.MAX_VALUE); - } - - if (cmd.hasOption(OPT_KEY_WINDOW)) { - keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW), - 0, Integer.MAX_VALUE); - } - - if (cmd.hasOption(OPT_MULTIGET)) { - multiGetBatchSize = parseInt(cmd.getOptionValue(OPT_MULTIGET), - 0, Integer.MAX_VALUE); - } - - System.out.println("Multi-gets (value of 1 means no multigets): " + multiGetBatchSize); - System.out.println("Percent of keys to verify: " + verifyPercent); - System.out.println("Reader threads: " + numReaderThreads); - } - - numTables = 1; - if (cmd.hasOption(NUM_TABLES)) { - numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE); - } - - numRegionsPerServer = DEFAULT_NUM_REGIONS_PER_SERVER; - if (cmd.hasOption(OPT_NUM_REGIONS_PER_SERVER)) { - numRegionsPerServer = Integer.parseInt(cmd.getOptionValue(OPT_NUM_REGIONS_PER_SERVER)); - } - - regionReplication = 1; - if (cmd.hasOption(OPT_REGION_REPLICATION)) { - regionReplication = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICATION)); - } - - regionReplicaId = -1; - if (cmd.hasOption(OPT_REGION_REPLICA_ID)) { - regionReplicaId = Integer.parseInt(cmd.getOptionValue(OPT_REGION_REPLICA_ID)); - } - } - - private void parseColumnFamilyOptions(CommandLine cmd) { - String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING); - dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null : - DataBlockEncoding.valueOf(dataBlockEncodingStr); - - String compressStr = cmd.getOptionValue(OPT_COMPRESSION); - compressAlgo = compressStr == null ? Compression.Algorithm.NONE : - Compression.Algorithm.valueOf(compressStr); - - String bloomStr = cmd.getOptionValue(OPT_BLOOM); - bloomType = bloomStr == null ? BloomType.ROW : - BloomType.valueOf(bloomStr); - - inMemoryCF = cmd.hasOption(OPT_INMEMORY); - if (cmd.hasOption(OPT_ENCRYPTION)) { - cipher = Encryption.getCipher(conf, cmd.getOptionValue(OPT_ENCRYPTION)); - } - - } - - public void initTestTable() throws IOException { - Durability durability = Durability.USE_DEFAULT; - if (deferredLogFlush) { - durability = Durability.ASYNC_WAL; - } - - HBaseTestingUtility.createPreSplitLoadTestTable(conf, tableName, - getColumnFamilies(), compressAlgo, dataBlockEncodingAlgo, numRegionsPerServer, - regionReplication, durability); - applyColumnFamilyOptions(tableName, getColumnFamilies()); - } - - @Override - protected int doWork() throws IOException { - if (numTables > 1) { - return parallelLoadTables(); - } else { - return loadTable(); - } - } - - protected int loadTable() throws IOException { - if (cmd.hasOption(OPT_ZK_QUORUM)) { - conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM)); - } - if (cmd.hasOption(OPT_ZK_PARENT_NODE)) { - conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, cmd.getOptionValue(OPT_ZK_PARENT_NODE)); - } - - if (isInitOnly) { - LOG.info("Initializing only; no reads or writes"); - initTestTable(); - return 0; - } - - if (!isSkipInit) { - initTestTable(); - } - LoadTestDataGenerator dataGen = null; - if (cmd.hasOption(OPT_GENERATOR)) { - String[] clazzAndArgs = cmd.getOptionValue(OPT_GENERATOR).split(COLON); - dataGen = getLoadGeneratorInstance(clazzAndArgs[0]); - String[] args; - if (dataGen instanceof LoadTestDataGeneratorWithACL) { - LOG.info("Using LoadTestDataGeneratorWithACL"); - if (User.isHBaseSecurityEnabled(conf)) { - LOG.info("Security is enabled"); - authnFileName = clazzAndArgs[1]; - superUser = clazzAndArgs[2]; - userNames = clazzAndArgs[3]; - args = Arrays.copyOfRange(clazzAndArgs, 2, clazzAndArgs.length); - Properties authConfig = new Properties(); - authConfig.load(this.getClass().getClassLoader().getResourceAsStream(authnFileName)); - try { - addAuthInfoToConf(authConfig, conf, superUser, userNames); - } catch (IOException exp) { - LOG.error(exp); - return EXIT_FAILURE; - } - userOwner = User.create(loginAndReturnUGI(conf, superUser)); - } else { - superUser = clazzAndArgs[1]; - userNames = clazzAndArgs[2]; - args = Arrays.copyOfRange(clazzAndArgs, 1, clazzAndArgs.length); - userOwner = User.createUserForTesting(conf, superUser, new String[0]); - } - } else { - args = clazzAndArgs.length == 1 ? new String[0] : Arrays.copyOfRange(clazzAndArgs, 1, - clazzAndArgs.length); - } - dataGen.initialize(args); - } else { - // Default DataGenerator is MultiThreadedAction.DefaultDataGenerator - dataGen = new MultiThreadedAction.DefaultDataGenerator(minColDataSize, maxColDataSize, - minColsPerKey, maxColsPerKey, families); - } - - if (userOwner != null) { - LOG.info("Granting permissions for user " + userOwner.getShortName()); - Permission.Action[] actions = { - Permission.Action.ADMIN, Permission.Action.CREATE, - Permission.Action.READ, Permission.Action.WRITE }; - try { - AccessControlClient.grant(ConnectionFactory.createConnection(conf), - tableName, userOwner.getShortName(), null, null, actions); - } catch (Throwable e) { - LOG.fatal("Error in granting permission for the user " + userOwner.getShortName(), e); - return EXIT_FAILURE; - } - } - - if (userNames != null) { - // This will be comma separated list of expressions. - String users[] = userNames.split(","); - User user = null; - for (String userStr : users) { - if (User.isHBaseSecurityEnabled(conf)) { - user = User.create(loginAndReturnUGI(conf, userStr)); - } else { - user = User.createUserForTesting(conf, userStr, new String[0]); - } - } - } - - if (isWrite) { - if (userOwner != null) { - writerThreads = new MultiThreadedWriterWithACL(dataGen, conf, tableName, userOwner); - } else { - String writerClass = null; - if (cmd.hasOption(OPT_WRITER)) { - writerClass = cmd.getOptionValue(OPT_WRITER); - } else { - writerClass = MultiThreadedWriter.class.getCanonicalName(); - } - - writerThreads = getMultiThreadedWriterInstance(writerClass, dataGen); - } - writerThreads.setMultiPut(isMultiPut); - } - - if (isUpdate) { - if (userOwner != null) { - updaterThreads = new MultiThreadedUpdaterWithACL(dataGen, conf, tableName, updatePercent, - userOwner, userNames); - } else { - String updaterClass = null; - if (cmd.hasOption(OPT_UPDATER)) { - updaterClass = cmd.getOptionValue(OPT_UPDATER); - } else { - updaterClass = MultiThreadedUpdater.class.getCanonicalName(); - } - updaterThreads = getMultiThreadedUpdaterInstance(updaterClass, dataGen); - } - updaterThreads.setBatchUpdate(isBatchUpdate); - updaterThreads.setIgnoreNonceConflicts(ignoreConflicts); - } - - if (isRead) { - if (userOwner != null) { - readerThreads = new MultiThreadedReaderWithACL(dataGen, conf, tableName, verifyPercent, - userNames); - } else { - String readerClass = null; - if (cmd.hasOption(OPT_READER)) { - readerClass = cmd.getOptionValue(OPT_READER); - } else { - readerClass = MultiThreadedReader.class.getCanonicalName(); - } - readerThreads = getMultiThreadedReaderInstance(readerClass, dataGen); - } - readerThreads.setMaxErrors(maxReadErrors); - readerThreads.setKeyWindow(keyWindow); - readerThreads.setMultiGetBatchSize(multiGetBatchSize); - readerThreads.setRegionReplicaId(regionReplicaId); - } - - if (isUpdate && isWrite) { - LOG.info("Concurrent write/update workload: making updaters aware of the " + - "write point"); - updaterThreads.linkToWriter(writerThreads); - } - - if (isRead && (isUpdate || isWrite)) { - LOG.info("Concurrent write/read workload: making readers aware of the " + - "write point"); - readerThreads.linkToWriter(isUpdate ? updaterThreads : writerThreads); - } - - if (isWrite) { - System.out.println("Starting to write data..."); - writerThreads.start(startKey, endKey, numWriterThreads); - } - - if (isUpdate) { - LOG.info("Starting to mutate data..."); - System.out.println("Starting to mutate data..."); - // TODO : currently append and increment operations not tested with tags - // Will update this aftet it is done - updaterThreads.start(startKey, endKey, numUpdaterThreads); - } - - if (isRead) { - System.out.println("Starting to read data..."); - readerThreads.start(startKey, endKey, numReaderThreads); - } - - if (isWrite) { - writerThreads.waitForFinish(); - } - - if (isUpdate) { - updaterThreads.waitForFinish(); - } - - if (isRead) { - readerThreads.waitForFinish(); - } - - boolean success = true; - if (isWrite) { - success = success && writerThreads.getNumWriteFailures() == 0; - } - if (isUpdate) { - success = success && updaterThreads.getNumWriteFailures() == 0; - } - if (isRead) { - success = success && readerThreads.getNumReadErrors() == 0 - && readerThreads.getNumReadFailures() == 0; - } - return success ? EXIT_SUCCESS : EXIT_FAILURE; - } - - private LoadTestDataGenerator getLoadGeneratorInstance(String clazzName) throws IOException { - try { - Class<?> clazz = Class.forName(clazzName); - Constructor<?> constructor = clazz.getConstructor(int.class, int.class, int.class, int.class, - byte[][].class); - return (LoadTestDataGenerator) constructor.newInstance(minColDataSize, maxColDataSize, - minColsPerKey, maxColsPerKey, families); - } catch (Exception e) { - throw new IOException(e); - } - } - - private MultiThreadedWriter getMultiThreadedWriterInstance(String clazzName - , LoadTestDataGenerator dataGen) throws IOException { - try { - Class<?> clazz = Class.forName(clazzName); - Constructor<?> constructor = clazz.getConstructor( - LoadTestDataGenerator.class, Configuration.class, TableName.class); - return (MultiThreadedWriter) constructor.newInstance(dataGen, conf, tableName); - } catch (Exception e) { - throw new IOException(e); - } - } - - private MultiThreadedUpdater getMultiThreadedUpdaterInstance(String clazzName - , LoadTestDataGenerator dataGen) throws IOException { - try { - Class<?> clazz = Class.forName(clazzName); - Constructor<?> constructor = clazz.getConstructor( - LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class); - return (MultiThreadedUpdater) constructor.newInstance( - dataGen, conf, tableName, updatePercent); - } catch (Exception e) { - throw new IOException(e); - } - } - - private MultiThreadedReader getMultiThreadedReaderInstance(String clazzName - , LoadTestDataGenerator dataGen) throws IOException { - try { - Class<?> clazz = Class.forName(clazzName); - Constructor<?> constructor = clazz.getConstructor( - LoadTestDataGenerator.class, Configuration.class, TableName.class, double.class); - return (MultiThreadedReader) constructor.newInstance(dataGen, conf, tableName, verifyPercent); - } catch (Exception e) { - throw new IOException(e); - } - } - - public static byte[] generateData(final Random r, int length) { - byte [] b = new byte [length]; - int i = 0; - - for(i = 0; i < (length-8); i += 8) { - b[i] = (byte) (65 + r.nextInt(26)); - b[i+1] = b[i]; - b[i+2] = b[i]; - b[i+3] = b[i]; - b[i+4] = b[i]; - b[i+5] = b[i]; - b[i+6] = b[i]; - b[i+7] = b[i]; - } - - byte a = (byte) (65 + r.nextInt(26)); - for(; i < length; i++) { - b[i] = a; - } - return b; - } - public static void main(String[] args) { - new LoadTestTool().doStaticMain(args); - } - - /** - * When NUM_TABLES is specified, the function starts multiple worker threads - * which individually start a LoadTestTool instance to load a table. Each - * table name is in format <tn>_<index>. For example, "-tn test -num_tables 2" - * , table names will be "test_1", "test_2" - * - * @throws IOException - */ - private int parallelLoadTables() - throws IOException { - // create new command args - String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME); - String[] newArgs = null; - if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) { - newArgs = new String[cmdLineArgs.length + 2]; - newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME; - newArgs[1] = LoadTestTool.DEFAULT_TABLE_NAME; - System.arraycopy(cmdLineArgs, 0, newArgs, 2, cmdLineArgs.length); - } else { - newArgs = cmdLineArgs; - } - - int tableNameValueIndex = -1; - for (int j = 0; j < newArgs.length; j++) { - if (newArgs[j].endsWith(OPT_TABLE_NAME)) { - tableNameValueIndex = j + 1; - } else if (newArgs[j].endsWith(NUM_TABLES)) { - // change NUM_TABLES to 1 so that each worker loads one table - newArgs[j + 1] = "1"; - } - } - - // starting to load multiple tables - List<WorkerThread> workers = new ArrayList<>(); - for (int i = 0; i < numTables; i++) { - String[] workerArgs = newArgs.clone(); - workerArgs[tableNameValueIndex] = tableName + "_" + (i+1); - WorkerThread worker = new WorkerThread(i, workerArgs); - workers.add(worker); - LOG.info(worker + " starting"); - worker.start(); - } - - // wait for all workers finish - LOG.info("Waiting for worker threads to finish"); - for (WorkerThread t : workers) { - try { - t.join(); - } catch (InterruptedException ie) { - IOException iie = new InterruptedIOException(); - iie.initCause(ie); - throw iie; - } - checkForErrors(); - } - - return EXIT_SUCCESS; - } - - // If an exception is thrown by one of worker threads, it will be - // stored here. - protected AtomicReference<Throwable> thrown = new AtomicReference<>(); - - private void workerThreadError(Throwable t) { - thrown.compareAndSet(null, t); - } - - /** - * Check for errors in the writer threads. If any is found, rethrow it. - */ - private void checkForErrors() throws IOException { - Throwable thrown = this.thrown.get(); - if (thrown == null) return; - if (thrown instanceof IOException) { - throw (IOException) thrown; - } else { - throw new RuntimeException(thrown); - } - } - - class WorkerThread extends Thread { - private String[] workerArgs; - - WorkerThread(int i, String[] args) { - super("WorkerThread-" + i); - workerArgs = args; - } - - @Override - public void run() { - try { - int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs); - if (ret != 0) { - throw new RuntimeException("LoadTestTool exit with non-zero return code."); - } - } catch (Exception ex) { - LOG.error("Error in worker thread", ex); - workerThreadError(ex); - } - } - } - - private void addAuthInfoToConf(Properties authConfig, Configuration conf, String owner, - String userList) throws IOException { - List<String> users = new ArrayList(Arrays.asList(userList.split(","))); - users.add(owner); - for (String user : users) { - String keyTabFileConfKey = "hbase." + user + ".keytab.file"; - String principalConfKey = "hbase." + user + ".kerberos.principal"; - if (!authConfig.containsKey(keyTabFileConfKey) || !authConfig.containsKey(principalConfKey)) { - throw new IOException("Authentication configs missing for user : " + user); - } - } - for (String key : authConfig.stringPropertyNames()) { - conf.set(key, authConfig.getProperty(key)); - } - LOG.debug("Added authentication properties to config successfully."); - } - - public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username) - throws IOException { - String hostname = InetAddress.getLocalHost().getHostName(); - String keyTabFileConfKey = "hbase." + username + ".keytab.file"; - String keyTabFileLocation = conf.get(keyTabFileConfKey); - String principalConfKey = "hbase." + username + ".kerberos.principal"; - String principal = SecurityUtil.getServerPrincipal(conf.get(principalConfKey), hostname); - if (keyTabFileLocation == null || principal == null) { - LOG.warn("Principal or key tab file null for : " + principalConfKey + ", " - + keyTabFileConfKey); - } - UserGroupInformation ugi = - UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation); - return ugi; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java index 1d2e9a6..6550baa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java @@ -49,7 +49,7 @@ import org.apache.hadoop.util.StringUtils; /** * Common base class for reader and writer parts of multi-thread HBase load - * test ({@link LoadTestTool}). + * test (See LoadTestTool). */ public abstract class MultiThreadedAction { private static final Log LOG = LogFactory.getLog(MultiThreadedAction.class); http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java index 1e7e341..e951175 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedReaderWithACL.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; import org.apache.hadoop.security.UserGroupInformation; @@ -121,7 +122,7 @@ public class MultiThreadedReaderWithACL extends MultiThreadedReader { UserGroupInformation realUserUgi; if(!users.containsKey(userNames[mod])) { if(User.isHBaseSecurityEnabled(conf)) { - realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + realUserUgi = HBaseKerberosUtils.loginAndReturnUGI(conf, userNames[mod]); } else { realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); } http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java index 40e23fb..9d9bb63 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedUpdaterWithACL.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.test.LoadTestDataGenerator; import org.apache.hadoop.security.UserGroupInformation; @@ -138,7 +139,7 @@ public class MultiThreadedUpdaterWithACL extends MultiThreadedUpdater { try { if (!users.containsKey(userNames[mod])) { if (User.isHBaseSecurityEnabled(conf)) { - realUserUgi = LoadTestTool.loginAndReturnUGI(conf, userNames[mod]); + realUserUgi = HBaseKerberosUtils.loginAndReturnUGI(conf, userNames[mod]); } else { realUserUgi = UserGroupInformation.createRemoteUser(userNames[mod]); } http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java index 6beb2e6..7972855 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/RestartMetaTest.java @@ -81,7 +81,7 @@ public class RestartMetaTest extends AbstractHBaseTool { // start the writers LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator( minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, - LoadTestTool.DEFAULT_COLUMN_FAMILY); + HFileTestUtil.DEFAULT_COLUMN_FAMILY); MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, TABLE_NAME); writer.setMultiPut(true); writer.start(startKey, endKey, numThreads); @@ -101,7 +101,7 @@ public class RestartMetaTest extends AbstractHBaseTool { // create tables if needed HBaseTestingUtility.createPreSplitLoadTestTable(conf, TABLE_NAME, - LoadTestTool.DEFAULT_COLUMN_FAMILY, Compression.Algorithm.NONE, + HFileTestUtil.DEFAULT_COLUMN_FAMILY, Compression.Algorithm.NONE, DataBlockEncoding.NONE); LOG.debug("Loading data....\n\n"); @@ -143,8 +143,8 @@ public class RestartMetaTest extends AbstractHBaseTool { @Override protected void addOptions() { addOptWithArg(OPT_NUM_RS, "Number of Region Servers"); - addOptWithArg(LoadTestTool.OPT_DATA_BLOCK_ENCODING, - LoadTestTool.OPT_DATA_BLOCK_ENCODING_USAGE); + addOptWithArg(HFileTestUtil.OPT_DATA_BLOCK_ENCODING, + HFileTestUtil.OPT_DATA_BLOCK_ENCODING_USAGE); } @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java deleted file mode 100644 index 51dc238..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionSizeCalculator.java +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.util; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HRegionLocation; -import org.apache.hadoop.hbase.RegionLoad; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.testclassification.MiscTests; -import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import static org.apache.hadoop.hbase.HConstants.DEFAULT_REGIONSERVER_PORT; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; - -@Category({MiscTests.class, SmallTests.class}) -public class TestRegionSizeCalculator { - - private Configuration configuration = new Configuration(); - private final long megabyte = 1024L * 1024L; - private final ServerName sn = ServerName.valueOf("local-rs", DEFAULT_REGIONSERVER_PORT, - ServerName.NON_STARTCODE); - - @Test - public void testSimpleTestCase() throws Exception { - - RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3"); - - Admin admin = mockAdmin( - mockRegion("region1", 123), - mockRegion("region3", 1232), - mockRegion("region2", 54321) - ); - - RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); - - assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes())); - assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes())); - assertEquals(1232 * megabyte, calculator.getRegionSize("region3".getBytes())); - // if regionCalculator does not know about a region, it should return 0 - assertEquals(0 * megabyte, calculator.getRegionSize("otherTableRegion".getBytes())); - - assertEquals(3, calculator.getRegionSizeMap().size()); - } - - - /** - * When size of region in megabytes is larger than largest possible integer there could be - * error caused by lost of precision. - * */ - @Test - public void testLargeRegion() throws Exception { - - RegionLocator regionLocator = mockRegionLocator("largeRegion"); - - Admin admin = mockAdmin( - mockRegion("largeRegion", Integer.MAX_VALUE) - ); - - RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); - - assertEquals(((long) Integer.MAX_VALUE) * megabyte, calculator.getRegionSize("largeRegion".getBytes())); - } - - /** When calculator is disabled, it should return 0 for each request.*/ - @Test - public void testDisabled() throws Exception { - String regionName = "cz.goout:/index.html"; - RegionLocator table = mockRegionLocator(regionName); - - Admin admin = mockAdmin( - mockRegion(regionName, 999) - ); - - //first request on enabled calculator - RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin); - assertEquals(999 * megabyte, calculator.getRegionSize(regionName.getBytes())); - - //then disabled calculator. - configuration.setBoolean(RegionSizeCalculator.ENABLE_REGIONSIZECALCULATOR, false); - RegionSizeCalculator disabledCalculator = new RegionSizeCalculator(table, admin); - assertEquals(0 * megabyte, disabledCalculator.getRegionSize(regionName.getBytes())); - - assertEquals(0, disabledCalculator.getRegionSizeMap().size()); - } - - /** - * Makes some table with given region names. - * */ - private RegionLocator mockRegionLocator(String... regionNames) throws IOException { - RegionLocator mockedTable = Mockito.mock(RegionLocator.class); - when(mockedTable.getName()).thenReturn(TableName.valueOf("sizeTestTable")); - List<HRegionLocation> regionLocations = new ArrayList<>(regionNames.length); - when(mockedTable.getAllRegionLocations()).thenReturn(regionLocations); - - for (String regionName : regionNames) { - HRegionInfo info = Mockito.mock(HRegionInfo.class); - when(info.getRegionName()).thenReturn(regionName.getBytes()); - regionLocations.add(new HRegionLocation(info, sn)); - } - - return mockedTable; - } - - /** - * Creates mock returning RegionLoad info about given servers. - */ - private Admin mockAdmin(RegionLoad... regionLoadArray) throws Exception { - Admin mockAdmin = Mockito.mock(Admin.class); - Map<byte[], RegionLoad> regionLoads = new TreeMap<>(Bytes.BYTES_COMPARATOR); - for (RegionLoad regionLoad : regionLoadArray) { - regionLoads.put(regionLoad.getName(), regionLoad); - } - when(mockAdmin.getConfiguration()).thenReturn(configuration); - when(mockAdmin.getRegionLoad(sn, TableName.valueOf("sizeTestTable"))).thenReturn(regionLoads); - return mockAdmin; - } - - /** - * Creates mock of region with given name and size. - * - * @param fileSizeMb number of megabytes occupied by region in file store in megabytes - * */ - private RegionLoad mockRegion(String regionName, int fileSizeMb) { - RegionLoad region = Mockito.mock(RegionLoad.class); - when(region.getName()).thenReturn(regionName.getBytes()); - when(region.getNameAsString()).thenReturn(regionName); - when(region.getStorefileSizeMB()).thenReturn(fileSizeMb); - return region; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java index bf7bf45..2deba00 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/test/LoadTestDataGenerator.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hbase.util.test; import java.io.IOException; +import java.util.Random; import java.util.Set; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -56,6 +57,28 @@ public abstract class LoadTestDataGenerator { this.kvGenerator = new LoadTestKVGenerator(minValueSize, maxValueSize); } + public static byte[] generateData(final Random r, int length) { + byte [] b = new byte [length]; + int i = 0; + + for(i = 0; i < (length-8); i += 8) { + b[i] = (byte) (65 + r.nextInt(26)); + b[i+1] = b[i]; + b[i+2] = b[i]; + b[i+3] = b[i]; + b[i+4] = b[i]; + b[i+5] = b[i]; + b[i+6] = b[i]; + b[i+7] = b[i]; + } + + byte a = (byte) (65 + r.nextInt(26)); + for(; i < length; i++) { + b[i] = a; + } + return b; + } + /** * initialize the LoadTestDataGenerator * http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties b/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties deleted file mode 100644 index 6fca96a..0000000 --- a/hbase-server/src/test/resources/org/apache/hadoop/hbase/PerformanceEvaluation_Counter.properties +++ /dev/null @@ -1,28 +0,0 @@ -# ResourceBundle properties file for Map-Reduce counters - -#/** -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -CounterGroupName= HBase Performance Evaluation -ELAPSED_TIME.name= Elapsed time in milliseconds -ROWS.name= Row count -# ResourceBundle properties file for Map-Reduce counters - -CounterGroupName= HBase Performance Evaluation -ELAPSED_TIME.name= Elapsed time in milliseconds -ROWS.name= Row count http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format b/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format deleted file mode 100755 index 762ddd7..0000000 Binary files a/hbase-server/src/test/resources/org/apache/hadoop/hbase/mapreduce/exportedTableIn94Format and /dev/null differ http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-spark/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-spark/pom.xml b/hbase-spark/pom.xml index aa47947..3761d59 100644 --- a/hbase-spark/pom.xml +++ b/hbase-spark/pom.xml @@ -492,6 +492,10 @@ <type>test-jar</type> </dependency> <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-mapreduce</artifactId> + </dependency> + <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java ---------------------------------------------------------------------- diff --git a/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java b/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java index 93cd939..bfacbe8 100644 --- a/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java +++ b/hbase-spark/src/test/java/org/apache/hadoop/hbase/spark/TestJavaHBaseContext.java @@ -27,7 +27,6 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index be0be99..f973cd7 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ <modules> <module>hbase-replication</module> + <module>hbase-mapreduce</module> <module>hbase-resource-bundle</module> <module>hbase-server</module> <module>hbase-thrift</module> @@ -1582,6 +1583,18 @@ <scope>test</scope> </dependency> <dependency> + <artifactId>hbase-mapreduce</artifactId> + <groupId>org.apache.hbase</groupId> + <version>${project.version}</version> + </dependency> + <dependency> + <artifactId>hbase-mapreduce</artifactId> + <groupId>org.apache.hbase</groupId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <artifactId>hbase-endpoint</artifactId> <groupId>org.apache.hbase</groupId> <version>${project.version}</version> http://git-wip-us.apache.org/repos/asf/hbase/blob/59d03410/src/main/asciidoc/_chapters/ops_mgt.adoc ---------------------------------------------------------------------- diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc index 6181b13..f96cd6c 100644 --- a/src/main/asciidoc/_chapters/ops_mgt.adoc +++ b/src/main/asciidoc/_chapters/ops_mgt.adoc @@ -2478,7 +2478,7 @@ void rename(Admin admin, String oldTableName, TableName newTableName) { RegionServer Grouping (A.K.A `rsgroup`) is an advanced feature for partitioning regionservers into distinctive groups for strict isolation. It should only be used by users who are sophisticated enough to understand the -full implications and have a sufficient background in managing HBase clusters. +full implications and have a sufficient background in managing HBase clusters. It was developed by Yahoo! and they run it at scale on their large grid cluster. See link:http://www.slideshare.net/HBaseCon/keynote-apache-hbase-at-yahoo-scale[HBase at Yahoo! Scale]. @@ -2491,20 +2491,20 @@ rsgroup at a time. By default, all tables and regionservers belong to the APIs. A custom balancer implementation tracks assignments per rsgroup and makes sure to move regions to the relevant regionservers in that rsgroup. The rsgroup information is stored in a regular HBase table, and a zookeeper-based read-only -cache is used at cluster bootstrap time. +cache is used at cluster bootstrap time. -To enable, add the following to your hbase-site.xml and restart your Master: +To enable, add the following to your hbase-site.xml and restart your Master: [source,xml] ---- - <property> - <name>hbase.coprocessor.master.classes</name> - <value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value> - </property> - <property> - <name>hbase.master.loadbalancer.class</name> - <value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value> - </property> + <property> + <name>hbase.coprocessor.master.classes</name> + <value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value> + </property> + <property> + <name>hbase.master.loadbalancer.class</name> + <value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value> + </property> ---- Then use the shell _rsgroup_ commands to create and manipulate RegionServer @@ -2514,7 +2514,7 @@ rsgroup commands available in the hbase shell type: [source, bash] ---- hbase(main):008:0> help ârsgroupâ - Took 0.5610 seconds + Took 0.5610 seconds ---- High level, you create a rsgroup that is other than the `default` group using @@ -2531,8 +2531,8 @@ Here is example using a few of the rsgroup commands. To add a group, do as foll [source, bash] ---- - hbase(main):008:0> add_rsgroup 'my_group' - Took 0.5610 seconds + hbase(main):008:0> add_rsgroup 'my_group' + Took 0.5610 seconds ---- @@ -2556,11 +2556,11 @@ ERROR: org.apache.hadoop.hbase.exceptions.UnknownProtocolException: No registere ==== Add a server (specified by hostname + port) to the just-made group using the -_move_servers_rsgroup_ command as follows: +_move_servers_rsgroup_ command as follows: [source, bash] ---- - hbase(main):010:0> move_servers_rsgroup 'my_group',['k.att.net:51129'] + hbase(main):010:0> move_servers_rsgroup 'my_group',['k.att.net:51129'] ---- .Hostname and Port vs ServerName