This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 0c3f7b201426b2740bc1d4fe53ee4951919c6061 Author: Puneet Ahuja <[email protected]> AuthorDate: Mon Aug 17 18:52:30 2026 +0530 SOLR-17841: Use faster DocSetCollector when multiThreaded=true (#4724) Improved multiThreaded=true performance when a docset is needed (e.g. faceting). Co-authored-by: David Smiley <[email protected]> (cherry picked from commit 3beb0dc5f2814fb607aa458fb4274453dbf4ec12) --- .../unreleased/SOLR-17841-mt-docset-collector.yml | 9 +++ .../apache/solr/search/MultiThreadedSearcher.java | 93 ++++++---------------- .../org/apache/solr/search/SolrIndexSearcher.java | 4 +- .../solr/search/TestMultiThreadedSearcher.java | 31 ++++++++ 4 files changed, 65 insertions(+), 72 deletions(-) diff --git a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml new file mode 100644 index 00000000000..3bd48c04e88 --- /dev/null +++ b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml @@ -0,0 +1,9 @@ +title: > + Improved multiThreaded=true performance when a docset is needed (e.g. faceting). +type: changed +authors: + - name: Puneet Ahuja + nick: punAhuja +links: + - name: SOLR-17841 + url: https://issues.apache.org/jira/browse/SOLR-17841 diff --git a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java index 3ad0712ff1a..75b8a61341a 100644 --- a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java @@ -22,14 +22,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.LinkedList; import java.util.concurrent.ExecutionException; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.search.TaskExecutor; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.search.TopFieldDocs; @@ -119,11 +117,14 @@ public class MultiThreadedSearcher { return new SearchResult(scoreMode, ret); } - static boolean allowMT(DelegatingCollector postFilter, QueryCommand cmd) { + static boolean allowMT(DelegatingCollector postFilter, QueryCommand cmd, TaskExecutor executor) { // TODO: it's unclear if segmentTerminateEarly is truly incompatible but // since it has to appropriately denote partial results this needs to be // investigated/tested before we can remove this check (perhaps for 9.8). - return postFilter == null && !cmd.getSegmentTerminateEarly() && cmd.getMultiThreaded(); + return postFilter == null + && !cmd.getSegmentTerminateEarly() + && cmd.getMultiThreaded() + && executor != null; } static class MaxScoreResult { @@ -134,67 +135,6 @@ public class MultiThreadedSearcher { } } - static class FixedBitSetCollector extends SimpleCollector { - @SuppressWarnings("JdkObsolete") - private final LinkedList<FixedBitSet> bitSets = new LinkedList<>(); - - @SuppressWarnings("JdkObsolete") - private final LinkedList<Integer> skipWords = new LinkedList<>(); - - @SuppressWarnings("JdkObsolete") - private final LinkedList<Integer> skipBits = new LinkedList<>(); - - FixedBitSetCollector() {} - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.bitSets.add(null); // lazy allocate when collecting document(s) - this.skipWords.add(context.docBase / 64); - this.skipBits.add(context.docBase % 64); - } - - @Override - public void collect(int doc) throws IOException { - FixedBitSet bitSet = this.bitSets.getLast(); - final int idx = this.skipBits.getLast() + doc; - - final int numWords = FixedBitSet.bits2words(idx + 1); // +1 to ensure minimum 1 word - - if (bitSet == null) { - this.bitSets.removeLast(); - bitSet = new FixedBitSet(numWords * 64); - this.bitSets.addLast(bitSet); - - } else if (bitSet.getBits().length < numWords) { - FixedBitSet smallerBitSet = this.bitSets.removeLast(); - bitSet = new FixedBitSet(numWords * 64); - bitSet.xor(smallerBitSet); - this.bitSets.addLast(bitSet); - } - - bitSet.set(idx); - } - - void update(FixedBitSet allBitSet) { - final long[] allBits = allBitSet.getBits(); - for (int bs_idx = 0; bs_idx < this.bitSets.size(); ++bs_idx) { - final FixedBitSet itBitSet = this.bitSets.get(bs_idx); - if (itBitSet != null) { - final int skipWords = this.skipWords.get(bs_idx); - final long[] itBits = itBitSet.getBits(); - for (int idx = 0; idx < itBits.length && skipWords + idx < allBits.length; ++idx) { - allBits[skipWords + idx] ^= itBits[idx]; - } - } - } - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - } - static class SearchResult { final ScoreMode scoreMode; private final Object[] result; @@ -283,8 +223,7 @@ public class MultiThreadedSearcher { @Override public Collector newCollector() throws IOException { - // TODO: add to firstCollectors here? or if not have comment w.r.t. why not adding - return new FixedBitSetCollector(); + return new DocSetCollector(maxDoc); } @Override @@ -292,12 +231,26 @@ public class MultiThreadedSearcher { public Object reduce(Collection collectors) throws IOException { final FixedBitSet reduced = new FixedBitSet(maxDoc); for (Object collector : collectors) { - if (collector instanceof FixedBitSetCollector fixedBitSetCollector) { - fixedBitSetCollector.update(reduced); + if (collector instanceof EarlyTerminatingCollector earlyTerminatingCollector) { + collector = earlyTerminatingCollector.getDelegate(); + } + if (collector instanceof DocSetCollector docSetCollector) { + mergeDocSetIntoFixedBitSet(docSetCollector.getDocSet(), reduced); } } return reduced; } + + private static void mergeDocSetIntoFixedBitSet(DocSet docSet, FixedBitSet reduced) { + if (docSet instanceof BitDocSet bitDocSet) { + reduced.or(bitDocSet.getBits()); + } else { + DocIterator iter = docSet.iterator(); + while (iter.hasNext()) { + reduced.set(iter.nextDoc()); + } + } + } } private class TopDocsCM implements CollectorManager<Collector, Object> { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 8098b39f4a1..33d86a7de77 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1982,7 +1982,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI } final TopDocs topDocs; final ScoreMode scoreModeUsed; - if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd)) { + if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, getTaskExecutor())) { log.trace("SINGLE THREADED search, skipping collector manager in getDocListNC"); final TopDocsCollector<?> topCollector = buildTopDocsCollector(len, cmd); MaxScoreCollector maxScoreCollector = null; @@ -2093,7 +2093,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI qr.setNextCursorMark(cmd.getCursorMark()); } else { final TopDocs topDocs; - if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd)) { + if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, getTaskExecutor())) { log.trace("SINGLE THREADED search, skipping collector manager in getDocListAndSetNC"); @SuppressWarnings({"rawtypes"}) diff --git a/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java index 2b83390158f..54325a8730a 100644 --- a/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java +++ b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java @@ -123,6 +123,37 @@ public class TestMultiThreadedSearcher extends SolrTestCaseJ4 { }); } + /** Multi-threaded DocSet collection must match single-threaded for the same query. */ + public void testMultiThreadedDocSetMatchesSingleThreaded() throws Exception { + h.getCore() + .withSearcher( + searcher -> { + assertTrue(searcher.getSlices().length > 1); + + final Query query = new TermQuery(new Term("field1_s", "xyzrareterm")); + final QueryCommand cmdSingle = new QueryCommand(); + cmdSingle.setQuery(query); + cmdSingle.setNeedDocSet(true); + cmdSingle.setLen(10); + cmdSingle.setMultiThreaded(false); + + final QueryCommand cmdMulti = new QueryCommand(); + cmdMulti.setQuery(query); + cmdMulti.setNeedDocSet(true); + cmdMulti.setLen(10); + cmdMulti.setMultiThreaded(true); + + final QueryResult singleThreaded = searcher.search(cmdSingle); + final QueryResult multiThreaded = searcher.search(cmdMulti); + + final DocSet stSet = singleThreaded.getDocListAndSet().docSet; + final DocSet mtSet = multiThreaded.getDocListAndSet().docSet; + assertEquals(stSet.size(), mtSet.size()); + assertTrue(DocSetUtil.equals(stSet, mtSet)); + return null; + }); + } + private static final class SimpleReRankQuery extends RankQuery { private Query q;
