This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 4a3bfa598e3 SOLR-18290: combined-query / RRF : support shards.rows
(#4546)
4a3bfa598e3 is described below
commit 4a3bfa598e39a5ece8e03b2154b37356b5228db3
Author: Sonu Sharma <[email protected]>
AuthorDate: Fri Jun 26 21:54:10 2026 +0530
SOLR-18290: combined-query / RRF : support shards.rows (#4546)
Enhance combined-query / RRF to support shards.rows param properly --
thereby supporting stable paging results
---
.../SOLR-17319-added_combined_query_RRF.yml | 3 +
.../handler/component/CombinedQueryComponent.java | 3 +-
.../component/CombinedQuerySolrCloudTest.java | 188 +++++++++++++--------
.../DistributedCombinedQueryComponentTest.java | 25 ++-
.../query-guide/pages/json-combined-query-dsl.adoc | 5 +
5 files changed, 150 insertions(+), 74 deletions(-)
diff --git a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml
b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml
index 17d64bf0233..a3f4f6177f8 100644
--- a/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml
+++ b/changelog/unreleased/SOLR-17319-added_combined_query_RRF.yml
@@ -6,3 +6,6 @@ authors:
links:
- name: SOLR-17319
url: https://issues.apache.org/jira/browse/SOLR-17319
+ - name: SOLR-18290
+ url: https://issues.apache.org/jira/browse/SOLR-18290
+
diff --git
a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
index 1e5c0b11aa1..28b4a80bb6e 100644
---
a/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
+++
b/solr/core/src/java/org/apache/solr/handler/component/CombinedQueryComponent.java
@@ -323,13 +323,14 @@ public class CombinedQueryComponent extends
QueryComponent implements SolrCoreAw
// Used to filter per-query docs so that RRF doesn't reintroduce docs
// excluded by collapse at the shard level.
Map<String, Set<Object>> combinedDocIdsPerShard =
HashMap.newHashMap(sreq.responses.size());
+ int perQueryQueueSize = Math.max(ss.getOffset() + ss.getCount(),
rb.shards_rows);
// TODO: to be parallelized outer loop
for (int queryIndex = 0; queryIndex < queriesToCombineKeys.length;
queryIndex++) {
int failedShardCount = 0;
long queryNumFound = 0;
long queryApproximateTotalHits = 0;
final ShardDocQueue queuePerQuery =
- newShardDocQueue(rb.req.getSearcher(), sortFields, ss.getOffset() +
ss.getCount());
+ newShardDocQueue(rb.req.getSearcher(), sortFields,
perQueryQueueSize);
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs = null;
NamedList<?> responseHeader;
diff --git
a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
index 39a7e69e23f..b4b72e0f2e5 100644
---
a/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
+++
b/solr/core/src/test/org/apache/solr/handler/component/CombinedQuerySolrCloudTest.java
@@ -126,16 +126,17 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
@Test
public void testSingleLexicalQuery() throws Exception {
prepareIndexDocs();
- QueryResponse rsp =
- query(
- CommonParams.JSON,
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:2^=10\"}}},"
- + "\"limit\":5,"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\"]}}",
- CommonParams.QT,
- "/search");
+ String jsonQuery =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:2^=10"}}
+ },
+ "limit": 5,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1"]}
+ }""";
+ QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT,
"/search");
assertEquals(1, rsp.getResults().size());
assertFieldValues(rsp.getResults(), id, "2");
}
@@ -149,12 +150,16 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
public void testMultipleLexicalQuery() throws Exception {
prepareIndexDocs();
String jsonQuery =
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3
OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4
OR 10^=2)\"}}},"
- + "\"limit\":5,"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}";
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 5,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
+ }""";
QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT,
"/search");
assertEquals(5, rsp.getResults().size());
assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3");
@@ -169,12 +174,17 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
public void testMultipleQueryWithSort() throws Exception {
prepareIndexDocs();
String jsonQuery =
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3
OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4
OR 10^=2)\"}}},"
- + "\"limit\":5,\"sort\":\"mod3_idv desc, score desc\""
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}";
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 5,
+ "sort": "mod3_idv desc, score desc",
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
+ }""";
QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT,
"/search");
assertEquals(5, rsp.getResults().size());
assertFieldValues(rsp.getResults(), id, "5", "2", "8", "7", "10");
@@ -188,40 +198,60 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
@Test
public void testHybridQueryWithPagination() throws Exception {
prepareIndexDocs();
- QueryResponse rsp =
- query(
- CommonParams.JSON,
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR
6^=3 OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR
7^=4 OR 10^=2)\"}}},"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",
- CommonParams.QT,
- "/search");
+ String jsonQueryAll =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
+ }""";
+ QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll,
CommonParams.QT, "/search");
+ // ideal ordering
assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10",
"8");
- rsp =
- query(
- CommonParams.JSON,
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR
6^=3 OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR
7^=4 OR 10^=2)\"}}},"
- + "\"limit\":4,"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",
- CommonParams.QT,
- "/search");
- assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");
- rsp =
- query(
- CommonParams.JSON,
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR
6^=3 OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR
7^=4 OR 10^=2)\"}}},"
- + "\"limit\":4,\"offset\":3,"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"combiner.query\":[\"lexical1\",\"lexical2\"]}}",
- CommonParams.QT,
- "/search");
+ String jsonQueryLimit2 =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 2,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
+ }""";
+ rsp = query(CommonParams.JSON, jsonQueryLimit2, CommonParams.QT,
"/search");
+ // assert proper ordering due to presence of shards.rows
+ assertFieldValues(rsp.getResults(), id, "7", "2");
+ String jsonQueryWithShardRows =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 2,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"], "shards.rows": 10}
+ }""";
+ rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT,
"/search");
+ assertFieldValues(rsp.getResults(), id, "5", "7");
+ // assert improper ordering due to lack of shards.rows
+ String jsonQueryPage =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 4,
+ "offset": 3,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
+ }""";
+ rsp = query(CommonParams.JSON, jsonQueryPage, CommonParams.QT, "/search");
assertEquals(4, rsp.getResults().size());
assertFieldValues(rsp.getResults(), id, "6", "3", "10", "8");
}
@@ -235,12 +265,22 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
public void testQueryWithFaceting() throws Exception {
prepareIndexDocs();
String jsonQuery =
- "{\"queries\":"
- + "{\"lexical\":{\"lucene\":{\"query\":\"id:(2^=2 OR 3^=1 OR 6^=2
OR 5^=1)\"}}},"
- + "\"limit\":3,\"offset\":1"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\",\"facet.mincount\":1,"
- + "\"combiner.query\":[\"lexical\"]}}";
+ """
+ {
+ "queries": {
+ "lexical": {"lucene": {"query": "id:(2^=2 OR 3^=1 OR 6^=2 OR
5^=1)"}}
+ },
+ "limit": 3,
+ "offset": 1,
+ "fields": ["id", "score", "title"],
+ "params": {
+ "combiner": true,
+ "facet": true,
+ "facet.field": "mod3_idv",
+ "facet.mincount": 1,
+ "combiner.query": ["lexical"]
+ }
+ }""";
QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT,
"/search");
assertEquals(3, rsp.getResults().size());
assertEquals(4, rsp.getResults().getNumFound());
@@ -256,14 +296,24 @@ public class CombinedQuerySolrCloudTest extends
AbstractFullDistribZkTestBase {
public void testQueriesWithFacetAndHighlights() throws Exception {
prepareIndexDocs();
String jsonQuery =
- "{\"queries\":"
- + "{\"lexical1\":{\"lucene\":{\"query\":\"id:(2^=4 OR 3^=2 OR 6^=3
OR 5^=1)\"}},"
- + "\"lexical2\":{\"lucene\":{\"query\":\"id:(8^=1 OR 5^=3 OR 7^=4
OR 10^=2)\"}}},"
- + "\"limit\":4,"
- + "\"fields\":[\"id\",\"score\",\"title\"],"
- +
"\"params\":{\"combiner\":true,\"facet\":true,\"facet.field\":\"mod3_idv\","
- + "\"combiner.query\":[\"lexical1\",\"lexical2\"], \"hl\": true,"
- + "\"hl.fl\": \"title\",\"hl.q\":\"test doc\"}}";
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(8^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 4,
+ "fields": ["id", "score", "title"],
+ "params": {
+ "combiner": true,
+ "facet": true,
+ "facet.field": "mod3_idv",
+ "combiner.query": ["lexical1", "lexical2"],
+ "hl": true,
+ "hl.fl": "title",
+ "hl.q": "test doc"
+ }
+ }""";
QueryResponse rsp = query(CommonParams.JSON, jsonQuery, CommonParams.QT,
"/search");
assertEquals(4, rsp.getResults().size());
assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");
diff --git
a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
index 17423bba672..50196cae1bc 100644
---
a/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
+++
b/solr/core/src/test/org/apache/solr/handler/component/DistributedCombinedQueryComponentTest.java
@@ -211,20 +211,36 @@ public class DistributedCombinedQueryComponentTest
extends BaseDistributedSearch
"params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
}""";
QueryResponse rsp = query(CommonParams.JSON, jsonQueryAll,
CommonParams.QT, "/search");
+ // ideal ordering
assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6", "3", "10",
"4");
- String jsonQueryLimit4 =
+ String jsonQueryLimit1 =
"""
{
"queries": {
"lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
"lexical2": {"lucene": {"query": "id:(4^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
},
- "limit": 4,
+ "limit": 1,
"fields": ["id", "score", "title"],
"params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"]}
}""";
- rsp = query(CommonParams.JSON, jsonQueryLimit4, CommonParams.QT,
"/search");
- assertFieldValues(rsp.getResults(), id, "5", "7", "2", "6");
+ rsp = query(CommonParams.JSON, jsonQueryLimit1, CommonParams.QT,
"/search");
+ // assert proper ordering due to presence of shards.rows
+ assertFieldValues(rsp.getResults(), id, "7");
+ String jsonQueryWithShardRows =
+ """
+ {
+ "queries": {
+ "lexical1": {"lucene": {"query": "id:(2^=4 OR 3^=2 OR 6^=3 OR
5^=1)"}},
+ "lexical2": {"lucene": {"query": "id:(4^=1 OR 5^=3 OR 7^=4 OR
10^=2)"}}
+ },
+ "limit": 1,
+ "fields": ["id", "score", "title"],
+ "params": {"combiner": true, "combiner.query": ["lexical1",
"lexical2"], "shards.rows": 10}
+ }""";
+ rsp = query(CommonParams.JSON, jsonQueryWithShardRows, CommonParams.QT,
"/search");
+ assertFieldValues(rsp.getResults(), id, "5");
+ // assert improper ordering due to lack of shards.rows
String jsonQueryPage =
"""
{
@@ -322,6 +338,7 @@ public class DistributedCombinedQueryComponentTest extends
BaseDistributedSearch
* @throws Exception if any unexpected error occurs during the test
execution.
*/
@Test
+ @ShardsFixed(num = 2)
public void testElevatedQueriesWithFacetAndHighlights() throws Exception {
prepareIndexDocs();
String jsonQuery =
diff --git
a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc
b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc
index de62c7a6d1b..61b7aed0b82 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/json-combined-query-dsl.adoc
@@ -76,6 +76,11 @@ The query structure is similar to JSON Query DSL except for
how multiple queries
`combiner.rrf.k` | Default: `60`::
The k parameter in the RRF algorithm.
+[NOTE]
+====
+Consider setting `shards.rows` to at least your deepest expected `start +
rows` to enlarge the per-shard candidate pool used for fusion, giving more
consistent ordering across pages.
+====
+
=== Example
Below is an example with sample JSON query payload:
