Updated Branches: refs/heads/apache-blur-0.2 4e5813433 -> 85ebe0a35
Adding tracing to the faceting code. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c6486046 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c6486046 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c6486046 Branch: refs/heads/apache-blur-0.2 Commit: c6486046cf31a590bd18439cc817ea3d99aa1e2c Parents: 4e58134 Author: Aaron McCurry <[email protected]> Authored: Thu Jan 9 20:41:35 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Jan 9 20:41:35 2014 -0500 ---------------------------------------------------------------------- .../blur/lucene/search/FacetExecutor.java | 48 ++++++++++++++------ .../blur/lucene/search/FacetQueryTest.java | 21 +++++++-- 2 files changed, 52 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c6486046/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java index 10ae351..587695b 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java @@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; +import org.apache.blur.trace.Trace; +import org.apache.blur.trace.Tracer; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.Collector; @@ -86,6 +88,8 @@ public class FacetExecutor { final OpenBitSet _bitSet; final Scorer[] _scorers; final AtomicReader _reader; + final String _readerStr; + final int _maxDoc; @Override public String toString() { @@ -97,34 +101,47 @@ public class FacetExecutor { _bitSet = new OpenBitSet(reader.maxDoc()); _scorers = scorers; _reader = reader; + _readerStr = _reader.toString(); + _maxDoc = _reader.maxDoc(); } void process(AtomicLongArray counts, long[] minimumsBeforeReturning) throws IOException { SimpleCollector col = new SimpleCollector(_bitSet); if (minimumsBeforeReturning == null) { - for (int i = 0; i < _scorers.length; i++) { - Scorer scorer = _scorers[i]; - if (scorer != null) { - scorer.score(col); - counts.addAndGet(i, col._hits); + Tracer trace = Trace.trace("processing facet - segment", Trace.param("reader", _readerStr), + Trace.param("maxDoc", _maxDoc), Trace.param("minimums", "NONE"), Trace.param("scorers", _scorers.length)); + try { + for (int i = 0; i < _scorers.length; i++) { + runFacet(counts, col, i); } - col._hits = 0; + } finally { + trace.done(); } } else { for (int i = 0; i < _scorers.length; i++) { long min = minimumsBeforeReturning[i]; long currentCount = counts.get(i); if (currentCount < min) { - Scorer scorer = _scorers[i]; - if (scorer != null) { - scorer.score(col); - counts.addAndGet(i, col._hits); - } - col._hits = 0; + runFacet(counts, col, i); } } } } + + private void runFacet(AtomicLongArray counts, SimpleCollector col, int i) throws IOException { + Scorer scorer = _scorers[i]; + if (scorer != null) { + Tracer traceInner = Trace.trace("processing facet - segment - scorer", Trace.param("scorer", scorer), + Trace.param("scorer.cost", scorer.cost())); + try { + scorer.score(col); + } finally { + traceInner.done(); + } + counts.addAndGet(i, col._hits); + } + col._hits = 0; + } } private final Map<Object, Info> _infoMap = new ConcurrentHashMap<Object, FacetExecutor.Info>(); @@ -188,7 +205,12 @@ public class FacetExecutor { public void processFacets(ExecutorService executor) throws IOException { if (!_processed) { - processInternal(executor); + Tracer trace = Trace.trace("processing facets"); + try { + processInternal(executor); + } finally { + trace.done(); + } _processed = true; } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c6486046/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java index 28e2c35..45d7354 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java @@ -24,9 +24,11 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.blur.concurrent.Executors; +import org.apache.blur.trace.LogTraceStorage; +import org.apache.blur.trace.Trace; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -71,7 +73,7 @@ public class FacetQueryTest { IndexSearcher indexSearcher = new IndexSearcher(reader); indexSearcher.search(facetQuery, 10); - ExecutorService executor = Executors.newCachedThreadPool(); + ExecutorService executor = getThreadPool(); facetExecutor.processFacets(executor); executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); @@ -83,6 +85,7 @@ public class FacetQueryTest { @Test public void testFacetQueryPerformance() throws IOException, InterruptedException { + Trace.setStorage(new LogTraceStorage(null)); int facetCount = 200; int docCount = 1000000; IndexReader reader = createIndex(docCount, facetCount, false); @@ -95,13 +98,19 @@ public class FacetQueryTest { ExecutorService executor = null; try { for (int t = 0; t < 5; t++) { - executor = Executors.newCachedThreadPool(); + executor = getThreadPool(); IndexSearcher indexSearcher = new IndexSearcher(reader, executor); FacetExecutor facetExecutor = new FacetExecutor(facets.length); FacetQuery facetQuery = new FacetQuery(new TermQuery(new Term("f1", "value")), facets, facetExecutor); long t1 = System.nanoTime(); indexSearcher.search(facetQuery, 10); + if (t == 4) { + Trace.setupTrace("unittest"); + } facetExecutor.processFacets(executor); + if (t == 4) { + Trace.tearDownTrace(); + } executor.shutdown(); executor.awaitTermination(10, TimeUnit.SECONDS); long t2 = System.nanoTime(); @@ -136,7 +145,7 @@ public class FacetQueryTest { ExecutorService executor = null; try { for (int t = 0; t < 5; t++) { - executor = Executors.newCachedThreadPool(); + executor = getThreadPool(); IndexSearcher indexSearcher = new IndexSearcher(reader, executor); FacetExecutor facetExecutor = new FacetExecutor(facets.length, minimumsBeforeReturning); FacetQuery facetQuery = new FacetQuery(new TermQuery(new Term("f1", "value")), facets, facetExecutor); @@ -158,6 +167,10 @@ public class FacetQueryTest { } } + private ExecutorService getThreadPool() { + return Executors.newThreadPool("unittest-facets", 10); + } + private IndexReader createIndex(int docCount, int facetFields, boolean ram) throws CorruptIndexException, LockObtainFailedException, IOException { Directory directory;
