Repository: kudu Updated Branches: refs/heads/branch-1.3.x 6cd1e0f9f -> 6139cdd69
ITBLL: avoid creating client many times during Generator step Previously ITBLL was creating a new client for each table to be created. This changes it to create the client at the top of 'run' and reuse the client. Change-Id: Ib3d9f5c974e1090b9367784fe8214a21a6b96b77 Reviewed-on: http://gerrit.cloudera.org:8080/6236 Reviewed-by: Jean-Daniel Cryans <[email protected]> Tested-by: Todd Lipcon <[email protected]> (cherry picked from commit 2f94fb665ca1e4fb750215f7522d06c5dfb0631f) Reviewed-on: http://gerrit.cloudera.org:8080/6241 Reviewed-by: Todd Lipcon <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/242fbfb7 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/242fbfb7 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/242fbfb7 Branch: refs/heads/branch-1.3.x Commit: 242fbfb7fb5aee33071b22c96ae8d8cf3ec52be5 Parents: 6cd1e0f Author: Todd Lipcon <[email protected]> Authored: Thu Mar 2 14:20:44 2017 -0800 Committer: Todd Lipcon <[email protected]> Committed: Fri Mar 3 18:38:13 2017 +0000 ---------------------------------------------------------------------- .../tools/IntegrationTestBigLinkedList.java | 83 +++++++++++--------- 1 file changed, 44 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/242fbfb7/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java ---------------------------------------------------------------------- diff --git a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java index 8efac49..19b2ba1 100644 --- a/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java +++ b/java/kudu-client-tools/src/main/java/org/apache/kudu/mapreduce/tools/IntegrationTestBigLinkedList.java @@ -27,6 +27,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -371,6 +372,9 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(Generator.class); + private CommandLineParser parser; + private KuduClient client; + static class GeneratorInputFormat extends InputFormat<BytesWritable,NullWritable> { static class GeneratorInputSplit extends InputSplit implements Writable { @@ -660,57 +664,58 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { public int run(int numMappers, long numNodes, int numTablets, Path tmpOutput, Integer width, Integer wrapMultiplier) throws Exception { - int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier); - if (ret > 0) { - return ret; + parser = new CommandLineParser(getConf()); + client = parser.getClient(); + try { + int ret = runRandomInputGenerator(numMappers, numNodes, tmpOutput, width, wrapMultiplier); + if (ret > 0) { + return ret; + } + return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier); + } finally { + client.close(); + client = null; } - return runGenerator(numMappers, numNodes, numTablets, tmpOutput, width, wrapMultiplier); } - protected void createTables(int numTablets) throws Exception { + private void createTables(int numTablets) throws Exception { createSchema(getTableName(getConf()), getTableSchema(), numTablets); createSchema(getHeadsTable(getConf()), getHeadsTableSchema(), numTablets); } - protected void createSchema(String tableName, Schema schema, int numTablets) throws Exception { - CommandLineParser parser = new CommandLineParser(getConf()); - KuduClient client = parser.getClient(); - try { - if (numTablets < 1) { - numTablets = 1; - } + private void createSchema(String tableName, Schema schema, int numTablets) throws Exception { + Preconditions.checkNotNull(client); + if (numTablets < 1) { + numTablets = 1; + } - if (client.tableExists(tableName)) { - return; - } + if (client.tableExists(tableName)) { + return; + } - CreateTableOptions builder = - new CreateTableOptions().setNumReplicas(parser.getNumReplicas()) - .setRangePartitionColumns(ImmutableList.of("key1", "key2")); - if (numTablets > 1) { - BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); - BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); - BigInteger step = max.multiply(BigInteger.valueOf(2)) - .divide(BigInteger.valueOf(numTablets)); - LOG.info(min.longValue()); - LOG.info(max.longValue()); - LOG.info(step.longValue()); - PartialRow splitRow = schema.newPartialRow(); - splitRow.addLong("key2", Long.MIN_VALUE); - for (int i = 1; i < numTablets; i++) { - long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue(); - LOG.info("key " + key); - splitRow.addLong("key1", key); - builder.addSplitRow(splitRow); - } + CreateTableOptions builder = + new CreateTableOptions().setNumReplicas(parser.getNumReplicas()) + .setRangePartitionColumns(ImmutableList.of("key1", "key2")); + if (numTablets > 1) { + BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); + BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); + BigInteger step = max.multiply(BigInteger.valueOf(2)) + .divide(BigInteger.valueOf(numTablets)); + LOG.info(min.longValue()); + LOG.info(max.longValue()); + LOG.info(step.longValue()); + PartialRow splitRow = schema.newPartialRow(); + splitRow.addLong("key2", Long.MIN_VALUE); + for (int i = 1; i < numTablets; i++) { + long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue(); + LOG.info("key " + key); + splitRow.addLong("key1", key); + builder.addSplitRow(splitRow); } - - client.createTable(tableName, schema, builder); - } finally { - // Done with this client. - client.shutdown(); } + + client.createTable(tableName, schema, builder); } public int runRandomInputGenerator(int numMappers, long numNodes, Path tmpOutput,
