cleaned up test class
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/71ab7cd5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/71ab7cd5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/71ab7cd5 Branch: refs/heads/master Commit: 71ab7cd5a207d47f60ee812e16a0c8f59832d9ef Parents: 3e7ea7f Author: gbarton <[email protected]> Authored: Sun Aug 26 11:19:20 2012 -0400 Committer: gbarton <[email protected]> Committed: Sun Aug 26 11:19:20 2012 -0400 ---------------------------------------------------------------------- .../blur/testsuite/CreateInsertQueryRepeating.java | 237 +++++++-------- 1 files changed, 106 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/71ab7cd5/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java index 70824d1..96aea40 100644 --- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java +++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java @@ -8,20 +8,13 @@ import java.util.Random; import java.util.UUID; import org.apache.thrift.TException; -import org.apache.thrift.async.AsyncMethodCallback; -import com.nearinfinity.blur.thrift.AsyncClientPool; import com.nearinfinity.blur.thrift.BlurClient; import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition; -import com.nearinfinity.blur.thrift.generated.Blur; -import com.nearinfinity.blur.thrift.generated.Blur.AsyncClient.mutateBatch_call; -import com.nearinfinity.blur.thrift.generated.Blur.AsyncClient.mutate_call; -import com.nearinfinity.blur.thrift.generated.Blur.AsyncIface; import com.nearinfinity.blur.thrift.generated.Blur.Iface; import com.nearinfinity.blur.thrift.generated.BlurException; import com.nearinfinity.blur.thrift.generated.BlurQuery; -import com.nearinfinity.blur.thrift.generated.ColumnDefinition; -import com.nearinfinity.blur.thrift.generated.ColumnFamilyDefinition; +import com.nearinfinity.blur.thrift.generated.BlurResults; import com.nearinfinity.blur.thrift.generated.RowMutation; import com.nearinfinity.blur.thrift.generated.RowMutationType; import com.nearinfinity.blur.thrift.generated.SimpleQuery; @@ -29,9 +22,15 @@ import com.nearinfinity.blur.thrift.generated.TableDescriptor; import static com.nearinfinity.blur.utils.BlurUtil.*; +/** + * Tests alot of things, mainly connecting to a blur cluster and slamming a bunch + * of rows in before querying for them. I like to use it as a load test. + * @author gman + * + */ public class CreateInsertQueryRepeating { - private static DecimalFormat df = new DecimalFormat("#,###,000.00"); + private DecimalFormat df = new DecimalFormat("#,###,000.00"); private static final char[] symbols = new char[36]; static { @@ -41,9 +40,37 @@ public class CreateInsertQueryRepeating { symbols[idx] = (char) ('a' + idx - 10); } - private static final Random random = new Random(); + private String table = "test1"; + private String host = "localhost"; + private Iface client = null; + + public CreateInsertQueryRepeating(String host, String table) throws BlurException, TException, IOException { + this.host = host; + this.table = table; + + //init + String connectionStr = host + ":40010"; + String cluster = "default"; + client = BlurClient.getClient(connectionStr); + + List<String> clusterList = client.shardClusterList(); + if(clusterList != null && clusterList.size() > 0) + cluster = clusterList.get(0); + else + throw new IOException("cannot find a cluster to use :("); + + System.out.println("using cluster: " + cluster); + + List<String> tableList = client.tableList(); + if (tableList == null || !tableList.contains(table)) + createTable(client, table, cluster); + else + System.out.println("table existed, did not create."); + } + + private final Random random = new Random(); - public static String randomString(int length) { + public String randomString(int length) { char[] buf = new char[length]; for (int idx = 0; idx < buf.length; ++idx) @@ -51,7 +78,7 @@ public class CreateInsertQueryRepeating { return new String(buf); } - public static void getClusters(Iface client) { + public void getClusters(Iface client) { try { List<String> shardClusterList = client.shardClusterList(); for (String cluster : shardClusterList) @@ -63,28 +90,22 @@ public class CreateInsertQueryRepeating { } } - public static void createTable(Iface client, String tableName, - String cluster) throws BlurException, TException { + public void createTable(Iface client, String tableName, String cluster) throws BlurException, TException { TableDescriptor td = new TableDescriptor(); td.analyzerDefinition = new AnalyzerDefinition(); - ColumnFamilyDefinition cfd = new ColumnFamilyDefinition(); - ColumnDefinition c = new ColumnDefinition( - "org.apache.lucene.analysis.standard.StandardAnalyzer", true, - null); - cfd.putToColumnDefinitions("cf1", c); - td.analyzerDefinition.putToColumnFamilyDefinitions("cf1", cfd); td.name = tableName; // TODO: doc doesnt say required, yet it barfs without it? td.cluster = cluster == null ? "default" : cluster; + //auto enable table td.isEnabled = true; - td.shardCount = 2; + //1 shard per server :) + td.shardCount = client.shardServerList(cluster).size(); td.readOnly = false; - // TODO: hardcodes bad - td.tableUri = "hdfs://localhost:8020/" + tableName; + // TODO: hardcodes bad, assuming NN on same node as BC + td.tableUri = "hdfs://" + host + ":8020/" + tableName; client.createTable(td); - // client.enableTable(tableName); System.out.println("table created"); } @@ -95,44 +116,69 @@ public class CreateInsertQueryRepeating { * @throws IOException */ public static void main(String[] args) throws BlurException, TException, IOException { - String connectionStr = "localhost:40010"; - final String cluster = "default"; - String uri = "hdfs://localhost:8020/test1"; - int shardCount = 1; - Iface client = BlurClient.getClient(connectionStr); - - List<String> tableList = client.tableList(); - if (tableList == null || !tableList.contains("test1")) - createTable(client, "test1", cluster); + String host = "localhost"; + String table = "test1"; + + if(args != null) { + if(args.length >= 1) + host = args[0]; + if(args.length == 2) + table = args[1]; + } + + CreateInsertQueryRepeating test = new CreateInsertQueryRepeating(host, table); +// System.out.println("Testing joins real quick"); +// test.testJoin(); +// System.out.println("test done"); + System.out.println("Starting load"); - loadupTable(client, "test1", 10); + test.loadupTable(100); System.out.println("Finshed load"); System.out.println("query time!"); - queryTable(client, "test1", 10000); + test.queryTable(50000); System.out.println("query done!"); - System.out.println("going into auto create test: "); - System.exit(0); - int i = 0; - while (i++ < 50) { - String tableName = UUID.randomUUID().toString(); - System.out.println("Creating [" + tableName + "]"); - createTable(client, cluster, uri, shardCount, tableName); - System.out.println("Loading [" + tableName + "]"); - loadTable(client, tableName); - System.out.println("Disabling [" + tableName + "]"); - disable(client, tableName); - System.out.println("Removing [" + tableName + "]"); - delete(client, tableName); - } } + @SuppressWarnings("unused") + private void testJoin() throws BlurException, TException { + RowMutation mutation = new RowMutation(); + mutation.table = table; + mutation.waitToBeVisible = true; + mutation.rowId = "row1"; + mutation.addToRecordMutations(newRecordMutation("cf1", + "recordid1", newColumn("col1","value1"))); + mutation.addToRecordMutations(newRecordMutation("cf1", + "recordid2", newColumn("col2","value2"))); + mutation.rowMutationType = RowMutationType.REPLACE_ROW; + client.mutate(mutation); + + List<String> joinTest = new ArrayList<String>(); + joinTest.add("+cf1.col1:value1"); + joinTest.add("+cf1.col2:value2"); + joinTest.add("+cf1.col1:value1 +cf1.col2:value2"); + joinTest.add("+(+cf1.col1:value1 nocf.nofield:somevalue) +(+cf1.col2.value2 nocf.nofield:somevalue)"); + joinTest.add("+(+cf1.col1:value1) +(cf1.bla:bla +cf1.col2.value2)"); + + for(String q : joinTest) + System.out.println(q + " hits: " + hits(client,table, q, true)); + } + + private static long hits(Iface client, String table, String queryStr, boolean superQuery) throws BlurException, TException { + BlurQuery bq = new BlurQuery(); + SimpleQuery sq = new SimpleQuery(); + sq.queryStr = queryStr; + sq.superQueryOn = superQuery; + bq.simpleQuery = sq; + BlurResults query = client.query(table, bq); + return query.totalResults; + } + // really only useful against the table that was filled via loadupTable - private static void queryTable(Iface client, String tableName, int times) - throws BlurException, TException { + public void queryTable(int times) throws BlurException, TException { long start = System.currentTimeMillis(); BlurQuery bq = new BlurQuery(); bq.fetch = 10; @@ -141,7 +187,7 @@ public class CreateInsertQueryRepeating { sq.queryStr = "numberField:" + random.nextInt(1000); sq.superQueryOn = true; bq.simpleQuery = sq; - client.query(tableName, bq); + client.query(table, bq); if (i % 1000 == 0) { System.out .println("queries: " @@ -160,23 +206,18 @@ public class CreateInsertQueryRepeating { } - private static void loadupTable(Iface client, String tableName, int rows) - throws BlurException, TException, IOException { - AsyncClientPool pool = new AsyncClientPool();//10, 30000); - AsyncIface poolClient = pool.getClient(Blur.AsyncIface.class, "localhost:40010"); - + public void loadupTable(int rows) throws BlurException, TException, IOException { long start = System.currentTimeMillis(); - List<RowMutation> mutates = new ArrayList<RowMutation>(); - long buildTotal = 0; + RowMutation mutation = new RowMutation(); for (int i = 1; i <= rows; i++) { long buildStart = System.currentTimeMillis(); - RowMutation mutation = new RowMutation(); - mutation.table = tableName; - mutation.waitToBeVisible = true; + mutation.clear(); + mutation.table = table; + mutation.waitToBeVisible = false; mutation.rowId = UUID.randomUUID().toString(); mutation.addToRecordMutations(newRecordMutation("test", "test-" + i, @@ -184,7 +225,7 @@ public class CreateInsertQueryRepeating { newColumn("numberField", i + ""), newColumn("fatTextField",randomString(1000)))); mutation.rowMutationType = RowMutationType.REPLACE_ROW; - + if (i % 50 == 0) { System.out.println("loaded: " + i + " around " + df.format((i / ((System.currentTimeMillis() - start+0.0)/1000) )) @@ -192,79 +233,13 @@ public class CreateInsertQueryRepeating { System.out.println("Total time: " + (System.currentTimeMillis()-start+0.0)/1000 + " Build time: " + ((buildTotal/1000)+0.0) + " " + buildTotal); } - - buildTotal += System.currentTimeMillis() - buildStart; -// mutates.add(mutation); - - poolClient.mutate(mutation, - new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() { - @Override - public void onComplete(mutate_call response) { - } + buildTotal += System.currentTimeMillis() - buildStart; - @Override - public void onError(Exception exception) { - exception.printStackTrace(); - } - }); + client.mutate(mutation); -// if(mutates.size() == 10) { -// pool.getClient(Blur.AsyncIface.class, "localhost:40010").mutateBatch(mutates, -// new AsyncMethodCallback<Blur.AsyncClient.mutateBatch_call>() { -// -// @Override -// public void onError(Exception exception) { -// exception.printStackTrace(); -// } -// -// @Override -// public void onComplete(mutateBatch_call response) { -// } -// }); -// mutates.clear(); -// } } System.out.println("loaded: " + rows + " around " + df.format((rows / ((System.currentTimeMillis() - start+0.0)/1000))) + " rows/s"); } - - private static void disable(Iface client, String tableName) - throws BlurException, TException { - client.disableTable(tableName); - } - - private static void delete(Iface client, String tableName) - throws BlurException, TException { - client.removeTable(tableName, true); - } - - private static void loadTable(Iface client, String tableName) - throws BlurException, TException { - RowMutation mutation = new RowMutation(); - mutation.table = tableName; - mutation.waitToBeVisible = true; - mutation.rowId = "test"; - mutation.addToRecordMutations(newRecordMutation("test", "test", - newColumn("test", "test"))); - mutation.rowMutationType = RowMutationType.REPLACE_ROW; - client.mutate(mutation); - } - - private static void createTable(Iface client, final String cluster, - String uri, int shardCount, String tableName) throws BlurException, - TException { - final TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.analyzerDefinition = new AnalyzerDefinition(); - tableDescriptor.cluster = cluster; - - tableDescriptor.name = tableName; - tableDescriptor.readOnly = false; - - tableDescriptor.shardCount = shardCount; - tableDescriptor.tableUri = uri + "/" + tableName; - - client.createTable(tableDescriptor); - } - }
