Some changes to output format for tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/4fd33272 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/4fd33272 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/4fd33272 Branch: refs/heads/master Commit: 4fd33272b90e748b7586801597b361d47ac59398 Parents: d4a4353 Author: Aaron McCurry <[email protected]> Authored: Tue Aug 28 20:41:53 2012 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Aug 28 20:41:53 2012 -0400 ---------------------------------------------------------------------- .../blur/testsuite/LoadDataContinuously.java | 27 ++++++++++---- .../testsuite/RandomSearchTableContinuously.java | 7 ++-- 2 files changed, 22 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4fd33272/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java index 53097d8..ff5a5a7 100644 --- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java +++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/LoadDataContinuously.java @@ -25,11 +25,13 @@ import com.nearinfinity.blur.thrift.generated.RowMutationType; public class LoadDataContinuously { private static Random random = new Random(); - private static List<String> words = new ArrayList<String>(); + public static List<String> words = new ArrayList<String>(); public static void main(String[] args) throws BlurException, TException, IOException { if (!(args.length == 8 || args.length == 9)) { - System.err.println(LoadDataContinuously.class.getName() + " <host1:port1,host2:port2> <table name> <WAL true|false> <# of columns per record> <# of records per row> <# of column families> <# of words per record> <time in seconds between reporting progress> <*optional path to word dictionary>"); + System.err + .println(LoadDataContinuously.class.getName() + + " <host1:port1,host2:port2> <table name> <WAL true|false> <# of columns per record> <# of records per row> <# of column families> <# of words per record> <time in seconds between reporting progress> <*optional path to word dictionary>"); System.exit(1); } if (args.length == 9) { @@ -37,7 +39,7 @@ public class LoadDataContinuously { } else { loadWords(null); } - + final Iface client = BlurClient.getClient(args[0]); final String table = args[1]; final boolean wal = Boolean.parseBoolean(args[2]); @@ -51,12 +53,14 @@ public class LoadDataContinuously { long s = start; long recordCountTotal = 0; long rowCount = 0; - + int batchSize = 100; - + List<RowMutation> batch = new ArrayList<RowMutation>(); - + long recordCount = 0; + long totalTime = 0; + long calls = 0; while (true) { long now = System.currentTimeMillis(); if (s + timeBetweenReporting < now) { @@ -64,9 +68,12 @@ public class LoadDataContinuously { double seconds = (now - s) / 1000.0; double avgRate = recordCountTotal / avgSeconds; double rate = recordCount / seconds; - System.out.println("Records indexed [" + recordCountTotal + "] Rows indexed [" + rowCount + "] at record rate [" + rate + "/s] with record avg rate [" + avgRate + "/s]"); + double latency = (totalTime / 1000000.0) / calls; + System.out.println(System.currentTimeMillis() + "," + recordCountTotal + "," + rowCount + "," + latency + "," + rate + "," + avgRate); s = now; recordCount = 0; + totalTime = 0; + calls = 0; } RowMutation mutation = new RowMutation(); @@ -80,7 +87,11 @@ public class LoadDataContinuously { } batch.add(mutation); if (batch.size() >= batchSize) { + long sm = System.nanoTime(); client.mutateBatch(batch); + long em = System.nanoTime(); + calls++; + totalTime += (em - sm); batch.clear(); } rowCount++; @@ -89,7 +100,7 @@ public class LoadDataContinuously { } } - private static void loadWords(String path) throws IOException { + public static void loadWords(String path) throws IOException { InputStream inputStream; if (path == null) { inputStream = LoadDataContinuously.class.getResourceAsStream("words.txt"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/4fd33272/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java index 13c09f2..9486fcc 100644 --- a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java +++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/RandomSearchTableContinuously.java @@ -51,7 +51,8 @@ public class RandomSearchTableContinuously { int count = 0; long resultCount = 0; Iface client = BlurClient.getClient(connectionStr); - for (int i = 0; i < numberOfSearchesPerPass; i++) { + int i; + for (i = 0; i < numberOfSearchesPerPass; i++) { long now = System.currentTimeMillis(); if (s + timeBetweenReporting < now) { double avgSeconds = (now - start) / 1000.0; @@ -59,8 +60,7 @@ public class RandomSearchTableContinuously { double avgRate = i / avgSeconds; double rate = count / seconds; double responseTimeAvg = (responseTime / (double) count) / 1000000.0; - System.out.println("Searches [" + i + "] at avg response time of [" + responseTimeAvg + " ms] at rate [" + rate + "/s] with avg rate [" + avgRate + "/s] results [" + resultCount + "]"); - System.out.println("Record Count [" + getCount(client,tableName) + "]"); + System.out.println(System.currentTimeMillis() + "," + i + "," + responseTimeAvg + "," + rate + "," + avgRate + "," + resultCount + "," + getCount(client,tableName)); s = now; responseTime = 0; count = 0; @@ -82,7 +82,6 @@ public class RandomSearchTableContinuously { responseTime += (qe - qs); count++; } - } private static long getCount(Iface client, String tableName) throws BlurException, TException {
