Adding more facting updates.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/831d22f3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/831d22f3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/831d22f3 Branch: refs/heads/console-v2 Commit: 831d22f38e8531b8455d5d4677cc71ca96d3e827 Parents: 9b96e81 Author: Aaron McCurry <[email protected]> Authored: Sat Mar 22 16:57:17 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sat Mar 22 16:57:17 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/thrift/BlurClusterTest.java | 47 +++++++++++++++-- .../blur/lucene/search/FacetExecutor.java | 53 +++++++++++++++++--- .../apache/blur/lucene/search/FacetQuery.java | 6 ++- 3 files changed, 93 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index b332f5c..41672e2 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -257,6 +257,10 @@ public class BlurClusterTest { } public void loadTable(String tableName) throws BlurException, TException, InterruptedException { + loadTable(tableName, 1); + } + + public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException { Iface client = getClient(); int maxFacetValue = 100; List<RowMutation> mutations = new ArrayList<RowMutation>(); @@ -265,7 +269,8 @@ public class BlurClusterTest { String rowId = UUID.randomUUID().toString(); RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, BlurThriftHelper.newColumn("test", "value"), - BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue)))); + BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))), + BlurThriftHelper.newColumn("facetFixed", "test")); RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); mutations.add(rowMutation); } @@ -281,7 +286,7 @@ public class BlurClusterTest { blurQueryRow.setCacheResult(false); BlurResults resultsRow = client.query(tableName, blurQueryRow); assertRowResults(resultsRow); - assertEquals(numberOfDocs, resultsRow.getTotalResults()); + assertEquals(numberOfDocs * pass, resultsRow.getTotalResults()); BlurQuery blurQueryRecord = new BlurQuery(); Query queryRecord = new Query(); @@ -290,7 +295,7 @@ public class BlurClusterTest { blurQueryRecord.setQuery(queryRecord); BlurResults resultsRecord = client.query(tableName, blurQueryRecord); assertRecordResults(resultsRecord); - assertEquals(numberOfDocs, resultsRecord.getTotalResults()); + assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults()); Schema schema = client.schema(tableName); assertFalse(schema.getFamilies().isEmpty()); @@ -342,6 +347,11 @@ public class BlurClusterTest { // assertRowResults(resultsRow); assertEquals(numberOfDocs, resultsRow.getTotalResults()); + List<Long> facetCounts = resultsRow.getFacetCounts(); + for (Long l : facetCounts) { + assertTrue(l >= 50); + } + System.out.println(resultsRow.getFacetCounts()); System.out.println(); @@ -349,6 +359,37 @@ public class BlurClusterTest { } @Test + public void testQueryWithFacetsWithMins() throws BlurException, TException, IOException, InterruptedException { + final String tableName = "testQueryWithFacetsWithMins"; + createTable(tableName); + int pass = 1; + loadTable(tableName, pass); + Iface client = getClient(); + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + // queryRow.setQuery("test.test:value"); + queryRow.setQuery("*"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setSelector(new Selector()); + blurQueryRow.addToFacets(new Facet("test.facetFixed:test", 50)); + + BlurResults resultsRow = client.query(tableName, blurQueryRow); + // assertRowResults(resultsRow); + System.out.println("Pass [" + pass + "]"); + assertEquals(numberOfDocs * pass, resultsRow.getTotalResults()); + + List<Long> facetCounts = resultsRow.getFacetCounts(); + for (Long l : facetCounts) { + System.out.println("Count [" + l + "]"); + assertTrue(l >= 50); + } + pass++; + + } + + @Test public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException { String tableName = "testBatchFetch"; createTable(tableName); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java index ed2abc7..913a318 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetExecutor.java @@ -39,13 +39,12 @@ import org.apache.blur.trace.Tracer; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.Collector; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.OpenBitSet; public class FacetExecutor { - private final Log LOG = LogFactory.getLog(FacetExecutor.class); + private static final Log LOG = LogFactory.getLog(FacetExecutor.class); static Comparator<Entry<Object, Info>> COMPARATOR = new Comparator<Entry<Object, Info>>() { @Override @@ -56,6 +55,10 @@ public class FacetExecutor { } }; + static class Finished extends IOException { + private static final long serialVersionUID = 1L; + } + static class SimpleCollector extends Collector { int _hits; @@ -73,8 +76,7 @@ public class FacetExecutor { } else { int nextSetBit = _bitSet.nextSetBit(doc); if (nextSetBit < 0) { - // Move to the end of the scorer. - _scorer.advance(DocIdSetIterator.NO_MORE_DOCS); + throw new Finished(); } else { int advance = _scorer.advance(nextSetBit); if (_bitSet.fastGet(advance)) { @@ -101,6 +103,28 @@ public class FacetExecutor { } + static class SimpleCollectorExitEarly extends SimpleCollector { + + private long _currentCount; + private final long _min; + + SimpleCollectorExitEarly(OpenBitSet bitSet, long currentCount, long min) { + super(bitSet); + _currentCount = currentCount; + _min = min; + } + + @Override + public void collect(int doc) throws IOException { + super.collect(doc); + _currentCount++; + if (_currentCount >= _min) { + throw new Finished(); + } + } + + } + static class Info { final OpenBitSet _bitSet; final Scorer[] _scorers; @@ -111,7 +135,7 @@ public class FacetExecutor { @Override public String toString() { - return "Info scorers length [" + _scorers.length + "] reader[" + _reader + "]"; + return "Info scorers length [" + _scorers.length + "] reader [" + _reader + "]"; } Info(AtomicReaderContext context, Scorer[] scorers, Lock[] locks) { @@ -125,12 +149,12 @@ public class FacetExecutor { } void process(AtomicLongArray counts, long[] minimumsBeforeReturning) throws IOException { - SimpleCollector col = new SimpleCollector(_bitSet); if (minimumsBeforeReturning == null) { Tracer trace = Trace.trace("processing facet - segment", Trace.param("reader", _readerStr), Trace.param("maxDoc", _maxDoc), Trace.param("minimums", "NONE"), Trace.param("scorers", _scorers.length)); try { for (int i = 0; i < _scorers.length; i++) { + SimpleCollector col = new SimpleCollector(_bitSet); runFacet(counts, col, i); } } finally { @@ -148,6 +172,8 @@ public class FacetExecutor { long min = minimumsBeforeReturning[id]; long currentCount = counts.get(id); if (currentCount < min) { + LOG.debug("Running facet, current count [{0}] min [{1}]", currentCount, min); + SimpleCollectorExitEarly col = new SimpleCollectorExitEarly(_bitSet, currentCount, min); runFacet(counts, col, id); } } finally { @@ -175,11 +201,16 @@ public class FacetExecutor { Tracer traceInner = Trace.trace("processing facet - segment - scorer", Trace.param("scorer", scorer), Trace.param("scorer.cost", scorer.cost())); try { + // new ExitScorer(scorer).score(col); scorer.score(col); + } catch (Finished e) { + // Do nothing, exiting early. } finally { traceInner.done(); } - counts.addAndGet(i, col._hits); + int hits = col._hits; + LOG.debug("Facet [{0}] result [{1}]", i, hits); + counts.addAndGet(i, hits); } col._hits = 0; } @@ -221,11 +252,16 @@ public class FacetExecutor { _infoMap.put(key, info); } else { AtomicReader reader = context.reader(); - LOG.info("Info about reader context [{0}] alread created, existing Info [{1}] current reader [{2}].", context, + LOG.warn("Info about reader context [{0}] already created, existing Info [{1}] current reader [{2}].", context, info, reader); } } + public boolean scorersAlreadyAdded(AtomicReaderContext context) { + Object key = getKey(context); + return _infoMap.containsKey(key); + } + private Object getKey(AtomicReaderContext context) { return context.reader().getCoreCacheKey(); } @@ -297,4 +333,5 @@ public class FacetExecutor { } } } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/831d22f3/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java index b813a0e..fbb342c 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/FacetQuery.java @@ -123,8 +123,10 @@ public class FacetQuery extends AbstractWrapperQuery { if (scorer == null) { return null; } - Scorer[] scorers = getScorers(context, true, topScorer, acceptDocs); - _executor.addScorers(context, scorers); + if (!_executor.scorersAlreadyAdded(context)) { + Scorer[] scorers = getScorers(context, true, topScorer, acceptDocs); + _executor.addScorers(context, scorers); + } return new FacetScorer(scorer, _executor, context); }
