This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 52c1ddb325cc00590cd2d1b7336be90841fc73fe Author: Abhishek Umarjikar <[email protected]> AuthorDate: Thu Jul 9 19:15:18 2026 +0530 SOLR-18197: Add root document query shortcut support to NestPathField (#4512) Querying root (top-level) documents no longer requires the verbose existence-negation idiom: Before: fq=*:* -_nest_path_:* After: fq=_nest_path_:\/ NestPathField now extends StrField instead of SortableTextField/CustomAnalyzer. Lucene's query parser bypasses getFieldQuery() for "tokenized" field; switching to an untokenized StrField fixed it. It's also simpler. The field is now a bit more strict about misconfiguration that was previously allowed. It doesn't support stored=true anymore but it's not needed anyway. (cherry picked from commit 83422bbb870f404daf7651c8201d20d3b2ecf360) --- .../SOLR_18197_nest_path_easy_of_use.yml | 10 ++ .../java/org/apache/solr/schema/NestPathField.java | 107 ++++++++++++++++----- .../solr/search/join/BlockJoinChildQParser.java | 2 +- .../solr/search/join/BlockJoinParentQParser.java | 21 ++-- .../org/apache/solr/search/QueryEqualityTest.java | 24 +++++ .../solr/update/TestNestedUpdateProcessor.java | 7 +- .../query-guide/pages/dense-vector-search.adoc | 6 +- .../pages/searching-nested-documents.adoc | 17 +++- 8 files changed, 150 insertions(+), 44 deletions(-) diff --git a/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.yml b/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.yml new file mode 100644 index 00000000000..646fe64ae83 --- /dev/null +++ b/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.yml @@ -0,0 +1,10 @@ +title: Add root document query shortcut support to NestPathField +type: added +authors: + - name: Abhishek Umarjikar + nick: abumarjikar + - name: David Smiley + nick: dsmiley +links: + - name: SOLR-18197 + url: https://issues.apache.org/jira/browse/SOLR-18197 diff --git a/solr/core/src/java/org/apache/solr/schema/NestPathField.java b/solr/core/src/java/org/apache/solr/schema/NestPathField.java index d34e532abbb..bac7fa74997 100644 --- a/solr/core/src/java/org/apache/solr/schema/NestPathField.java +++ b/solr/core/src/java/org/apache/solr/schema/NestPathField.java @@ -17,13 +17,18 @@ package org.apache.solr.schema; -import java.io.IOException; +import java.util.List; import java.util.Map; -import org.apache.lucene.analysis.core.KeywordTokenizerFactory; -import org.apache.lucene.analysis.custom.CustomAnalyzer; -import org.apache.lucene.analysis.pattern.PatternReplaceFilterFactory; -import org.apache.solr.analysis.TokenizerChain; +import java.util.regex.Pattern; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; /** * To be used for field {@link IndexSchema#NEST_PATH_FIELD_NAME} for enhanced nested doc @@ -34,32 +39,82 @@ import org.apache.solr.common.SolrException; * @see org.apache.solr.update.processor.NestedUpdateProcessorFactory * @since 8.0 */ -public class NestPathField extends SortableTextField { +public class NestPathField extends StrField { + + private static final Pattern ORDINAL_PATTERN = Pattern.compile("#\\d*"); @Override public void setArgs(IndexSchema schema, Map<String, String> args) { - args.putIfAbsent("stored", "false"); - args.putIfAbsent("multiValued", "false"); - args.putIfAbsent("omitTermFreqAndPositions", "true"); - args.putIfAbsent("omitNorms", "true"); - args.putIfAbsent("maxCharsForDocValues", "-1"); + args.putIfAbsent("stored", "false"); // flip a default + args.putIfAbsent("docValues", "true"); // flip a default; necessary for old schemas + args.putIfAbsent("multiValued", "false"); // flip a default; necessary for old schemas + args.putIfAbsent("uninvertible", "false"); // flip a default; necessary for old schemas super.setArgs(schema, args); + // Doesn't support these flags; perhaps others too. + // note: we could support STORED if truly useful but why bother given docValues. + restrictProps(STORED | MULTIVALUED | UNINVERTIBLE | TOKENIZED); + } + + /** + * {@inheritDoc} Overridden to manipulate the indexed form (AKA terms), but not the DV form. The + * value may look something like {@code /childA#0/childB#23} and we want to strip out the + * pound-digits aspects. + * + * <p>The terms index is used for block-join and docValues is used for returning nested docs in + * the same structure as given. + * + * @see #toInternal(String) + */ + @Override + public List<IndexableField> createFields(SchemaField field, Object value) { + // indexed or docValues or both may be false... albeit we strongly recommend both enabled and + // the testing of disabling either may be non-existent. + final IndexableField termField = createField(field, value); // calls toInternal + + if (!field.hasDocValues()) { // note: strongly recommended, however + return termField == null ? List.of() : List.of(termField); + } + + final BytesRef bytes = getBytesRef(value); // unmodified, unlike the indexed term + final IndexableField dvField = new SortedDocValuesField(field.getName(), bytes); + + return termField == null ? List.of(dvField) : List.of(termField, dvField); + } + + @Override + public String toInternal(String val) { + return ORDINAL_PATTERN.matcher(val).replaceAll(""); + } + + @Override + public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { + if (externalVal == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Field " + field.getName() + " missing value. Forgot `v` local-param?"); + } - // CustomAnalyzer is easy to use - CustomAnalyzer customAnalyzer; - try { - customAnalyzer = - CustomAnalyzer.builder(schema.getResourceLoader()) - .withDefaultMatchVersion(schema.getDefaultLuceneMatchVersion()) - .withTokenizer(KeywordTokenizerFactory.class) - .addTokenFilter( - PatternReplaceFilterFactory.class, "pattern", "#\\d*", "replace", "all") - .build(); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); // impossible? + if (externalVal.contains("\\")) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Field " + field.getName() + " query value contains backslashes ('\\')."); } - // Solr HTTP Schema APIs don't know about CustomAnalyzer so use TokenizerChain instead - setIndexAnalyzer(new TokenizerChain(customAnalyzer)); - // leave queryAnalyzer as literal + + if (externalVal.isEmpty() || "/".equals(externalVal)) { + return new BooleanQuery.Builder() + .add(MatchAllDocsQuery.INSTANCE, BooleanClause.Occur.MUST) + .add(field.getType().getExistenceQuery(parser, field), BooleanClause.Occur.MUST_NOT) + .build(); + } + + if (!externalVal.startsWith("/") || externalVal.endsWith("/")) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Field " + + field.getName() + + " query value must start with a forward slash ('/') and cannot contain a trailing slash."); + } + + return super.getFieldQuery(parser, field, externalVal); } } diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java index 0d7383dfc4f..b53d7d81680 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java @@ -90,7 +90,7 @@ public class BlockJoinChildQParser extends BlockJoinParentQParser { } // allParents filter: (*:* -{!prefix f="_nest_path_" v="<parentPath>/"}) - // For root: (*:* -_nest_path_:*) + // For root: {!field f=_nest_path_ v='/'} final Query allParentsFilter = buildAllParentsFilterFromPath(parentPath); // constrain the parent query to only match docs at exactly parentPath diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java index 6e7dbaf4c63..6cd34424b95 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java @@ -171,11 +171,11 @@ public class BlockJoinParentQParser extends FiltersQParser { if (parsedChildQuery.clauses().isEmpty()) { // i.e. all children // no block-join needed; just return all "parent" docs at this level - return wrapWithParentPathConstraint(parentPath, new MatchAllDocsQuery()); + return wrapWithParentPathConstraint(parentPath, MatchAllDocsQuery.INSTANCE); } // allParents filter: (*:* -{!prefix f="_nest_path_" v="<parentPath>/"}) - // For root: (*:* -_nest_path_:*) + // For root: {!field f=_nest_path_ v='/'} final Query allParentsFilter = buildAllParentsFilterFromPath(parentPath); // constrain child query: (+<original_child> +{!prefix f="_nest_path_" v="<parentPath>/"}) @@ -206,18 +206,19 @@ public class BlockJoinParentQParser extends FiltersQParser { * </ul> * * <p>Equivalent to: {@code (*:* -{!prefix f="_nest_path_" v="<parentPath>/"})} For root ({@code - * /}): {@code (*:* -_nest_path_:*)} + * /}): {@code {!field f=_nest_path_ v='/'}} */ protected Query buildAllParentsFilterFromPath(String parentPath) { - final Query excludeQuery; - if (parentPath.equals("/")) { - excludeQuery = newNestPathExistsQuery(); - } else { - excludeQuery = new PrefixQuery(new Term(IndexSchema.NEST_PATH_FIELD_NAME, parentPath + "/")); + if (parentPath.equals("/") || parentPath.isEmpty()) { + final SchemaField nestPathField = req.getSchema().getField(IndexSchema.NEST_PATH_FIELD_NAME); + return nestPathField.getType().getFieldQuery(this, nestPathField, parentPath); } + return new BooleanQuery.Builder() - .add(new MatchAllDocsQuery(), Occur.MUST) - .add(excludeQuery, Occur.MUST_NOT) + .add(MatchAllDocsQuery.INSTANCE, Occur.MUST) + .add( + new PrefixQuery(new Term(IndexSchema.NEST_PATH_FIELD_NAME, parentPath + "/")), + Occur.MUST_NOT) .build(); } diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java index ecad83f39de..b8895fe7b18 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.NamedMatches; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; @@ -2000,6 +2001,29 @@ public class QueryEqualityTest extends SolrTestCaseJ4 { "{!hash_range l='107347968' u='214695935' f='x_id'}"); } + @Test + public void testNestPathRootShortcut() throws Exception { + try (SolrQueryRequest req = req("df", "_nest_path_")) { + Query parsedQ = + assertQueryEqualsAndReturn( + null, req, "{!field f=_nest_path_ v=''}", "{!field f=_nest_path_}/"); + + var schemaField = req.getSchema().getField("_nest_path_"); + Query expectedQ = + new BooleanQuery.Builder() + .add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST) + .add( + schemaField.getType().getExistenceQuery(null, schemaField), + BooleanClause.Occur.MUST_NOT) + .build(); + + assertEquals( + "The root shortcut query did not form the expected match-all minus existence structure", + expectedQ, + parsedQ); + } + } + // Override req to add df param public static SolrQueryRequest req(String... q) { return SolrTestCaseJ4.req(q, "df", "text"); diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index fca48d5765a..d5482e52c3f 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -690,8 +690,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { "child_q", "(+" + inner_child_query + " +_nest_path_:*)"); } else { return params( - "q", - "{!parent which='(*:* -_nest_path_:*)'}(+" + inner_child_query + " +_nest_path_:*)"); + "q", "{!parent which=_nest_path_:\\/}(+" + inner_child_query + " +_nest_path_:*)"); } } // else... @@ -786,7 +785,9 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { } else { return params( "q", - "{!child of='(*:* -_nest_path_:*)'}(+" + inner_parent_query + " -_nest_path_:*)"); + "{!child of='{!field f=_nest_path_ v=/}'}(+" + + inner_parent_query + + " -_nest_path_:*)"); } } // else... diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index 381a33b5ee5..1eaf66c3825 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -539,7 +539,7 @@ Here is an example of a `knn` search using a `childrenOf`: [source,text] ?q={!knn f=vector topK=3 childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0] -&allParents=*:* -_nest_path_:* +&allParents={!field f=_nest_path_ v='/'} The search results retrieved are the k=3 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, each of them with a different parent. The 'childrenOf' parameter must return all valid parents to guarantee the correct functioning of the query. @@ -560,7 +560,7 @@ Here is an example of a `knn` search using a `parents.preFilter`: [source,text] ?q={!knn f=vector topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0] -&allParents=*:* -_nest_path_:* +&allParents={!field f=_nest_path_ v='/'} &someParents=color_s:RED The search results retrieved are the k=3 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, each of them with a different parent. Only the documents with a parent that satisfy the 'color_s:RED' condition are considered candidates for the ANN search. @@ -605,7 +605,7 @@ So you should query a multivalued vector fields following the same syntax: [source,text] ?q={!parent which=$allParents score=max v=$children.q} &children.q={!knn f=vector_multivalued topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0] -&allParents=*:* -_nest_path_:* +&allParents={!field f=_nest_path_ v='/'} &someParents=color_s:RED In terms of rendering the results, you need the child transformer if you want to output them flat (you can choose to only return the best vector per result or all vectors): diff --git a/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc b/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc index 81220d51318..879663b17ac 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc @@ -274,6 +274,21 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' - }} ---- +=== Root Document Query Shortcut + +When working with hierarchical nested documents, you may frequently need to isolate or filter your search results exclusively to top-level "root" documents (documents that do not belong to a parent path layout). + +Instead of using a verbose negative field existence filter (like `*:* -_nest_path_:*`), the `NestPathField` type supports an explicit root document query shortcut. Passing either a single forward slash (`/`) or an empty string (`''`) automatically constructs a query matching all documents while excluding any nested child paths: + +[source,text] +---- +# Targets only top-level root documents via the slash shortcut +fq={!field f=_nest_path_}/ + +# Alternative equivalent syntax using an empty parameter string +fq={!field f=_nest_path_ v=''} +---- + === Nested Vectors search through Block Join Query Parsers and Child Doc Transformer @@ -293,7 +308,7 @@ An example: [source,text] ?q={!parent which=$allParents score=max v=$children.q}& children.q={!knn f=vector topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0]& -allParents=*:* -_nest_path_:*& +allParents={!field f=_nest_path_ v=''}& someParents=color_s:RED& fl=id,score,vectors,vector,[child fl=vector childFilter=$children.q]
