Added a test for in bulk performance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/68cf6ed8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/68cf6ed8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/68cf6ed8 Branch: refs/heads/0.2-dev Commit: 68cf6ed8ff5153323b1421571c4be91c90c980af Parents: ce72e54 Author: Aaron McCurry <[email protected]> Authored: Sat Nov 24 21:47:06 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Nov 24 21:47:06 2012 -0500 ---------------------------------------------------------------------- .../org/apache/blur/thrift/ITBlurClusterTest.java | 55 +++++++++++++++ 1 files changed, 55 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/68cf6ed8/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java index 83d9ceb..28d1144 100644 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java @@ -190,4 +190,59 @@ public class ITBlurClusterTest { } assertEquals(SHARD_COUNT * 100, totalHits); } + + @Test + public void testAddDocumentsPerformance() throws BlurException, TException, InterruptedException, IOException { + Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test-add-table-perf"); + tableDescriptor.setShardCount(SHARD_COUNT); + tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-add-table-perf"); + client.createTable(tableDescriptor); + + List<Generation> generations = new ArrayList<Generation>(); + int length = 100000; + int batch = 100; + long t1 = System.nanoTime(); + for (int s = 0; s < SHARD_COUNT; s++) { + List<Document> documents = new ArrayList<Document>(); + for (int i = 0; i < length; i++) { + Document doc = new Document(); + doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()), TYPE.STRING, 1.0)); + doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING, 1.0)); + documents.add(doc); + if (documents.size() >= batch) { + MutateOptions options = new MutateOptions(); + options.setTable(tableDescriptor.getName()); + options.setShardIndex(s); + generations.addAll(client.addDocuments(options, documents)); + documents.clear(); + } + } + MutateOptions options = new MutateOptions(); + options.setTable(tableDescriptor.getName()); + options.setShardIndex(s); + generations.addAll(client.addDocuments(options, documents)); + } + long t2 = System.nanoTime(); + + client.blockUntilGenerationIsVisible(generations, true); + + long t3 = System.nanoTime(); + + System.out.println("Add time [" + (t2 - t1) / 1000000000.0 + "]"); + System.out.println("Block time [" + (t3 - t2) / 1000000000.0 + "]"); + + Session session = client.openReadSession(tableDescriptor.getName()); + QueryArgs queryArgs = new QueryArgs(); + Term term = new Term("value", "test"); + TermQuery query = new TermQuery(term); + queryArgs.setQuery(ThriftLuceneConversion.toBytes(query)); + List<TopFieldDocs> results = client.search(session, queryArgs); + long totalHits = 0; + for (TopFieldDocs fieldDocs : results) { + totalHits += fieldDocs.getTotalHits(); + } + assertEquals(SHARD_COUNT * length, totalHits); + } }
