This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit f7b0ec018b69b58be13cccc51d0b61fdb51f75e0 Author: Sonu Sharma <[email protected]> AuthorDate: Fri Jun 26 12:24:10 2026 -0400 SOLR-18290: combined-query / RRF : support shards.rows (#4546) Enhance combined-query / RRF to support shards.rows param properly -- thereby supporting stable paging results 9x backport changes for Java 11 --- .../SOLR-17319-added_combined_query_RRF.yml | 3 + .../handler/component/CombinedQueryComponent.java | 3 +- .../component/CombinedQuerySolrCloudTest.java | 141 +++++++++++---------- .../DistributedCombinedQueryComponentTest.java | 21 ++- .../query-guide/pages/json-combined-query-dsl.adoc | 5 + 5 files changed, 99 insertions(+), 74 deletions(-) diff --git a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml index 17d64bf0233..a3f4f6177f8 100644 --- a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml +++ b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml @@ -6,3 +6,6 @@ authors: links: - name: SOLR-17319 url: https://issues.apache.org/jira/browse/SOLR-17319 + - name: SOLR-18290 + url: https://issues.apache.org/jira/browse/SOLR-18290 + diff --git a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java index 699fe8d2fd8..6c76f314649 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java @@ -327,13 +327,14 @@ public class CombinedQueryComponent extends QueryComponent implements SolrCoreAw // excluded by collapse at the shard level. Map<String, Set<Object>> combinedDocIdsPerShard = CollectionUtil.newHashMap(sreq.responses.size()); + int perQueryQueueSize = Math.max(ss.getOffset() + ss.getCount(), rb.shards_rows); // TODO: to be parallelized outer loop for (int queryIndex = 0; queryIndex < queriesToCombineKeys.length; queryIndex++) { int failedShardCount = 0; long queryNumFound = 0; long queryApproximateTotalHits = 0; final ShardDocQueue queuePerQuery = - newShardDocQueue(rb.req.getSearcher(), sortFields, ss.getOffset() + ss.getCount()); + newShardDocQueue(rb.req.getSearcher(), sortFields, perQueryQueueSize); for (ShardResponse srsp : sreq.responses) { SolrDocumentList docs = null; NamedList<?> responseHeader; diff --git a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java index 3c5f56261ec..24772adedf4 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java @@ -126,16 +126,13 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { @Test public void testSingleLexicalQuery() throws Exception { prepareIndexDocs(); - QueryResponse rsp = - query( - CommonParams.JSON, - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:2^=10\"}}}," - + "\"limit\":5," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\"]}}", - CommonParams.QT, - "/search"); + String jsonQuery = + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:2^=10\"}}}," + + "\"limit\": 5," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\"]}}"; + QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search"); assertEquals(1, rsp.getResults().size()); assertFieldValues(rsp.getResults(), id, "2"); } @@ -149,12 +146,12 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { public void testMultipleLexicalQuery() throws Exception { prepareIndexDocs(); String jsonQuery = - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":5," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}"; + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 5," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"]}}"; QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search"); assertEquals(5, rsp.getResults().size()); assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3"); @@ -169,12 +166,13 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { public void testMultipleQueryWithSort() throws Exception { prepareIndexDocs(); String jsonQuery = - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":5,\"sort\":\"mod3_idv desc, score desc\"" - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}"; + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 5," + + "\"sort\": \"mod3_idv desc, score desc\"," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"]}}"; QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search"); assertEquals(5, rsp.getResults().size()); assertFieldValues(rsp.getResults(), id, "5", "2", "8", "7", "10"); @@ -188,40 +186,44 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { @Test public void testHybridQueryWithPagination() throws Exception { prepareIndexDocs(); - QueryResponse rsp = - query( - CommonParams.JSON, - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}", - CommonParams.QT, - "/search"); + String jsonQueryAll = + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"]}}"; + QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll, CommonParams.QT, "/search"); + // ideal ordering assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10", "8"); - rsp = - query( - CommonParams.JSON, - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":4," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}", - CommonParams.QT, - "/search"); - assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6"); - rsp = - query( - CommonParams.JSON, - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":4,\"offset\":3," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}", - CommonParams.QT, - "/search"); + String jsonQueryLimit2 = + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 2," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"]}}"; + rsp = query(CommonParams.JSON, jsonQueryLimit2, CommonParams.QT, "/search"); + // assert proper ordering due to presence of shards.rows + assertFieldValues(rsp.getResults(), id, "7", "2"); + String jsonQueryWithShardRows = + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 2," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"], \"shards.rows\": 10}}"; + rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT, "/search"); + assertFieldValues(rsp.getResults(), id, "5", "7"); + // assert improper ordering due to lack of shards.rows + String jsonQueryPage = + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 4," + + "\"offset\": 3," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"combiner.query\": [\"lexical1\", \"lexical2\"]}}"; + rsp = query(CommonParams.JSON, jsonQueryPage, CommonParams.QT, "/search"); assertEquals(4, rsp.getResults().size()); assertFieldValues(rsp.getResults(), id, "6", "3", "10", "8"); } @@ -235,12 +237,13 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { public void testQueryWithFaceting() throws Exception { prepareIndexDocs(); String jsonQuery = - "{\"queries\":" - + "{\"lexical\":{\"lucene\":{\"query\":\"id:(2^=2 OR 3^=1 OR 6^=2 OR 5^=1)\"}}}," - + "\"limit\":3,\"offset\":1" - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\",\"facet.mincount\":1," - + "\"combiner.query\":[\"lexical\"]}}"; + "{\"queries\": {" + + "\"lexical\": {\"lucene\": {\"query\": \"id:(2^=2 OR 3^=1 OR 6^=2 OR 5^=1)\"}}}," + + "\"limit\": 3," + + "\"offset\": 1," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"facet\": true, \"facet.field\": \"mod3_idv\"," + + "\"facet.mincount\": 1, \"combiner.query\": [\"lexical\"]}}"; QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search"); assertEquals(3, rsp.getResults().size()); assertEquals(4, rsp.getResults().getNumFound()); @@ -256,14 +259,14 @@ public class CombinedQuerySolrCloudTest extends AbstractFullDistribZkTestBase { public void testQueriesWithFacetAndHighlights() throws Exception { prepareIndexDocs(); String jsonQuery = - "{\"queries\":" - + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," - + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":4," - + "\"fields\":[\"id\",\"score\",\"title\"]," - + "\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\"," - + "\"combiner.query\":[\"lexical1\",\"lexical2\"], \"hl\": true," - + "\"hl.fl\": \"title\",\"hl.q\":\"test doc\"}}"; + "{\"queries\": {" + + "\"lexical1\": {\"lucene\": {\"query\": \"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\": {\"lucene\": {\"query\": \"id:(8^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\": 4," + + "\"fields\": [\"id\", \"score\", \"title\"]," + + "\"params\": {\"combiner\": true, \"facet\": true, \"facet.field\": \"mod3_idv\"," + + "\"combiner.query\": [\"lexical1\", \"lexical2\"]," + + "\"hl\": true, \"hl.fl\": \"title\", \"hl.q\": \"test doc\"}}"; QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT, "/search"); assertEquals(4, rsp.getResults().size()); assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6"); diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java index f45cee60cfc..4303a1eb238 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java @@ -199,16 +199,28 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch + "\"fields\":[\"id\",\"score\",\"title\"]," + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}"; QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll, CommonParams.QT, "/search"); + // ideal ordering assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10", "4"); - String jsonQueryLimit4 = + String jsonQueryLimit1 = "{\"queries\":" + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + "\"lexical2\":{\"lucene\":{\"query\":\"id:(4^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," - + "\"limit\":4," + + "\"limit\":1," + "\"fields\":[\"id\",\"score\",\"title\"]," + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}"; - rsp = query(CommonParams.JSON, jsonQueryLimit4, CommonParams.QT, "/search"); - assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6"); + rsp = query(CommonParams.JSON, jsonQueryLimit1, CommonParams.QT, "/search"); + // assert proper ordering due to presence of shards.rows + assertFieldValues(rsp.getResults(), id, "7"); + String jsonQueryWithShardRows = + "{\"queries\":" + + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," + + "\"lexical2\":{\"lucene\":{\"query\":\"id:(4^=1 OR 5^=3 OR 7^=4 OR 10^=2)\"}}}," + + "\"limit\":1," + + "\"fields\":[\"id\",\"score\",\"title\"]," + + "\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"],\"shards.rows\":10}}"; + rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT, "/search"); + assertFieldValues(rsp.getResults(), id, "5"); + // assert improper ordering due to lack of shards.rows String jsonQueryPage = "{\"queries\":" + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3 OR 5^=1)\"}}," @@ -281,6 +293,7 @@ public class DistributedCombinedQueryComponentTest extends BaseDistributedSearch * @throws Exception if any unexpected error occurs during the test execution. */ @Test + @ShardsFixed(num = 2) public void testElevatedQueriesWithFacetAndHighlights() throws Exception { prepareIndexDocs(); String jsonQuery = diff --git a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc index de62c7a6d1b..61b7aed0b82 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc @@ -76,6 +76,11 @@ The query structure is similar to JSON Query DSL except for how multiple queries `combiner.rrf.k` | Default: `60`:: The k parameter in the RRF algorithm. +[NOTE] +==== +Consider setting `shards.rows` to at least your deepest expected `start + rows` to enlarge the per-shard candidate pool used for fusion, giving more consistent ordering across pages. +==== + === Example Below is an example with sample JSON query payload:
