This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 2901f1c79854d526d0facdd7723fb7fbf6b86410 Author: David Smiley <[email protected]> AuthorDate: Tue Oct 27 12:27:16 2026 -0400 SOLR-3479: New [hl] document transformer for inline highlighting (#5126) Also works in conjunction with /export. Not with RTG. Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]> Co-authored-by: Nick Shanin <[email protected]> (cherry picked from commit c7c23508840a4c828283c76716902e5b1b6f9d2e) --- .../SOLR-3479-highlight-doc-transformer.yml | 7 + .../solr/highlight/UnifiedSolrHighlighter.java | 47 +++- .../transform/HighlightAugmenterFactory.java | 268 +++++++++++++++++++++ .../response/transform/TransformerFactory.java | 1 + .../apache/solr/cloud/TestRandomFlRTGCloud.java | 3 +- .../transform/TestHighlightAugmenterFactory.java | 262 ++++++++++++++++++++ .../query-guide/pages/document-transformers.adoc | 35 +++ .../modules/query-guide/pages/highlighting.adoc | 14 +- 8 files changed, 621 insertions(+), 16 deletions(-) diff --git a/changelog/unreleased/SOLR-3479-highlight-doc-transformer.yml b/changelog/unreleased/SOLR-3479-highlight-doc-transformer.yml new file mode 100644 index 00000000000..b7b188a0d31 --- /dev/null +++ b/changelog/unreleased/SOLR-3479-highlight-doc-transformer.yml @@ -0,0 +1,7 @@ +title: "New [hl] document transformer: highlight a field inline in each document, e.g. fl=id,body:[hl]" +type: added +authors: + - name: David Smiley +links: + - name: SOLR-3479 + url: https://issues.apache.org/jira/browse/SOLR-3479 diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java index e1314790197..0d37c7fd005 100644 --- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java +++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java @@ -136,6 +136,9 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf protected static final String SNIPPET_SEPARATOR = "\u0000"; + /** Default for {@link HighlightParams#SNIPPETS}. */ + public static final int DEFAULT_SNIPPETS = 1; + @Override public void init(PluginInfo info) {} @@ -157,7 +160,8 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf int maxPassages[] = new int[fieldNames.length]; for (int i = 0; i < fieldNames.length; i++) { - maxPassages[i] = params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, 1); + maxPassages[i] = + params.getFieldInt(fieldNames[i], HighlightParams.SNIPPETS, DEFAULT_SNIPPETS); } UnifiedHighlighter highlighter = getHighlighter(req); @@ -194,8 +198,7 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf if (snippet == null) { // TODO reuse logic of DefaultSolrHighlighter.alternateField } else { - // we used a special snippet separator char and we can now split on it. - summary.add(field, snippet.split(SNIPPET_SEPARATOR)); + summary.add(field, splitSnippets(snippet)); } } list.add(keys[i], summary); @@ -203,6 +206,15 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf return list; } + /** + * Splits a highlighted value formatted by {@link SolrExtendedUnifiedHighlighter} into its + * snippets. + */ + public static String[] splitSnippets(String highlighted) { + // a special snippet separator char is used unless hl.tag.ellipsis is specified + return highlighted.split(SNIPPET_SEPARATOR); + } + /** Converts solr's DocList to the int[] docIDs */ protected int[] toDocIDs(DocList docs) { int[] docIDs = new int[docs.size()]; @@ -238,8 +250,12 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf } } - /** From {@link #getHighlighter(org.apache.solr.request.SolrQueryRequest)}. */ - protected static class SolrExtendedUnifiedHighlighter extends UnifiedHighlighter { + /** + * A {@link UnifiedHighlighter} configured by Solr's highlighting params. + * + * @lucene.experimental + */ + public static class SolrExtendedUnifiedHighlighter extends UnifiedHighlighter { protected static final Predicate<String> NOT_REQUIRED_FIELD_MATCH_PREDICATE = s -> true; private final SolrIndexSearcher solrIndexSearcher; protected final SolrParams params; @@ -248,19 +264,24 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf protected final RTimerTree loadFieldValuesTimer; public SolrExtendedUnifiedHighlighter(SolrQueryRequest req) { - super(req.getSearcher(), req.getSchema().getIndexAnalyzer()); - this.solrIndexSearcher = req.getSearcher(); - this.params = req.getParams(); - this.schema = req.getSchema(); + this(req.getSearcher(), req.getSchema(), req.getParams(), req.getRequestTimer()); + } + + /** + * @param timerTree request timer to record field-loading time in; may be null + */ + public SolrExtendedUnifiedHighlighter( + SolrIndexSearcher searcher, IndexSchema schema, SolrParams params, RTimerTree timerTree) { + super(searcher, schema.getIndexAnalyzer()); + this.solrIndexSearcher = searcher; + this.params = params; + this.schema = schema; this.setMaxLength(params.getInt(HighlightParams.MAX_CHARS, DEFAULT_MAX_CHARS)); this.setCacheFieldValCharsThreshold( params.getInt( HighlightParams.CACHE_FIELD_VAL_CHARS_THRESHOLD, DEFAULT_CACHE_CHARS_THRESHOLD)); - final RTimerTree timerTree; - if (req.getRequestTimer() != null) { // It may be null if not used in a search context. - timerTree = req.getRequestTimer(); - } else { + if (timerTree == null) { // It may be null if not used in a search context. timerTree = new RTimerTree(); // since null checks are annoying } loadFieldValuesTimer = diff --git a/solr/core/src/java/org/apache/solr/response/transform/HighlightAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/HighlightAugmenterFactory.java new file mode 100644 index 00000000000..cb8836f1ecd --- /dev/null +++ b/solr/core/src/java/org/apache/solr/response/transform/HighlightAugmenterFactory.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.uhighlight.FieldHighlighter; +import org.apache.lucene.search.uhighlight.UnifiedHighlighter; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.HighlightParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.highlight.UnifiedSolrHighlighter; +import org.apache.solr.highlight.UnifiedSolrHighlighter.SolrExtendedUnifiedHighlighter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.ResultContext; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocIterationInfo; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; +import org.apache.solr.search.SyntaxError; + +/** + * Highlights one field of each document with the {@link UnifiedHighlighter}, adding the snippets as + * a list under the transformer's name, e.g. {@code body_hl:[hl f=body]}. Without {@code f}, the + * field is the transformer's name, e.g. {@code body:[hl]}. When the name is the field and nothing + * is highlighted, the field's original value is returned instead. + * + * <p>Other local params are {@code hl.*} params without the {@code hl.} prefix (e.g. {@code + * snippets}, {@code tag.pre}, {@code q}); they override the request's {@code hl.*} params, which + * otherwise apply. + */ +public class HighlightAugmenterFactory extends TransformerFactory { + + /** Local param naming the field to highlight. */ + public static final String FIELD_PARAM = "f"; + + private static final String HL_PREFIX = HighlightParams.HIGHLIGHT + "."; + + /** Names of the {@code hl.*} params in {@link HighlightParams}, without the prefix. */ + private static final Set<String> HL_PARAM_NAMES = + Arrays.stream(HighlightParams.class.getFields()) + .filter(f -> f.getType() == String.class && Modifier.isStatic(f.getModifiers())) + .map( + f -> { + try { + return (String) f.get(null); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + }) + .filter(name -> name.startsWith(HL_PREFIX)) + .map(name -> name.substring(HL_PREFIX.length())) + .collect(Collectors.toUnmodifiableSet()); + + @Override + public DocTransformer create(String display, SolrParams localParams, SolrQueryRequest req) { + String field = localParams.get(FIELD_PARAM, display); + SchemaField schemaField = req.getSchema().getFieldOrNull(field); + if (schemaField == null) { + throw new SolrException( + ErrorCode.BAD_REQUEST, + "[hl] needs a field to highlight, named by the '" + + FIELD_PARAM + + "' param or by its alias; not a field: " + + field); + } + + ModifiableSolrParams overrides = new ModifiableSolrParams(); + for (Iterator<String> it = localParams.getParameterNamesIterator(); it.hasNext(); ) { + String name = it.next(); + if (name.equals(FIELD_PARAM)) { + continue; + } + if (!HL_PARAM_NAMES.contains(name)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown [hl] param: " + name); + } + String[] values = localParams.getParams(name); + overrides.set(HL_PREFIX + name, values); + // per-field form too, so that it overrides a request's per-field param + overrides.set("f." + field + "." + HL_PREFIX + name, values); + } + SolrParams params = SolrParams.wrapDefaults(overrides, req.getParams()); + + boolean passThrough = display.equals(field); + + boolean singleValue = + !schemaField.multiValued() + && params.getFieldInt( + field, HighlightParams.SNIPPETS, UnifiedSolrHighlighter.DEFAULT_SNIPPETS) + == 1; + + return new HighlightAugmenter( + display, field, params, parseHighlightQuery(params, req), passThrough, singleValue); + } + + /** Parses {@code hl.q}, as the highlight component does; null if absent. */ + private static Query parseHighlightQuery(SolrParams params, SolrQueryRequest req) { + String hlq = params.get(HighlightParams.Q); + if (hlq == null) { + return null; + } + String parserName = + params.get( + HighlightParams.QPARSER, params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE)); + try { + return QParser.getParser(hlq, parserName, req).getHighlightQuery(); + } catch (SyntaxError e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + } + + static class HighlightAugmenter extends DocTransformer { + private final String name; + private final String field; + private final SolrParams params; + private final Query hlQuery; + private final boolean passThrough; + private final boolean singleValue; // output a value instead of a list + private final SolrReturnFields originalValueFields; + private SingleFieldHighlighter highlighter; // null if there's no query + + HighlightAugmenter( + String name, + String field, + SolrParams params, + Query hlQuery, + boolean passThrough, + boolean singleValue) { + this.name = name; + this.field = field; + this.params = params; + this.hlQuery = hlQuery; + this.passThrough = passThrough; + this.singleValue = singleValue; + this.originalValueFields = passThrough ? new SolrReturnFields(List.of(field), null) : null; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean needsSolrIndexSearcher() { + return true; + } + + @Override + public void setContext(ResultContext context) { + super.setContext(context); + Query query = hlQuery != null ? hlQuery : getMainHighlightQuery(context); + SolrIndexSearcher searcher = context.getSearcher(); + highlighter = + (query == null || searcher == null) + ? null + : new SingleFieldHighlighter(searcher, params, field, query); + } + + // Identity is intended: the context must come from this exact request and query. + @SuppressWarnings("ReferenceEquality") + private static Query getMainHighlightQuery(ResultContext context) { + SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); + ResponseBuilder rb = info == null ? null : info.getResponseBuilder(); + // A custom result context may have a different query from the current request. + if (rb == null + || rb.req != context.getRequest() + || rb.getQuery() != context.getQuery() + || rb.getQparser() == null) { + return context.getQuery(); + } + try { + return rb.getQparser().getHighlightQuery(); + } catch (SyntaxError e) { + throw new SolrException(ErrorCode.BAD_REQUEST, e); + } + } + + @Override + public void transform(SolrDocument doc, int docid, DocIterationInfo docInfo) + throws IOException { + if (docid < 0) { + return; // not from the index + } + Object highlighted = highlighter == null ? null : highlighter.highlight(docid); + List<String> snippets = + highlighted == null + ? List.of() + // a multi-valued field's value separator can yield empty snippets + : Arrays.stream(UnifiedSolrHighlighter.splitSnippets(highlighted.toString())) + .filter(s -> !s.isEmpty()) + .toList(); + if (!snippets.isEmpty()) { + doc.setField(name, singleValue ? snippets.getFirst() : snippets); + } else if (passThrough) { + Object original = + context.getDocFetcher().solrDoc(docid, originalValueFields).getFieldValue(field); + if (original != null) { + doc.setField(name, original); + } + } else if (!singleValue) { + doc.setField(name, List.of()); + } + } + } + + /** + * Highlights one field of one document at a time. Query analysis and per-field setup happen once, + * in the constructor. + */ + private static final class SingleFieldHighlighter extends SolrExtendedUnifiedHighlighter { + private final String[] fields; + private final FieldHighlighter fieldHighlighter; + + SingleFieldHighlighter( + SolrIndexSearcher searcher, SolrParams params, String field, Query query) { + super(searcher, searcher.getSchema(), params, null); + this.fields = new String[] {field}; + int maxPassages = + params.getFieldInt( + field, HighlightParams.SNIPPETS, UnifiedSolrHighlighter.DEFAULT_SNIPPETS); + this.fieldHighlighter = getFieldHighlighter(field, query, extractTerms(query), maxPassages); + } + + /** Returns the formatted highlight of the doc's field, or null if none. */ + Object highlight(int docId) throws IOException { + CharSequence content = + loadFieldValues(fields, DocIdSetIterator.range(docId, docId + 1), 0).getFirst()[0]; + if (content == null) { + return null; + } + List<LeafReaderContext> leaves = searcher.getIndexReader().leaves(); + LeafReaderContext leaf = leaves.get(ReaderUtil.subIndex(docId, leaves)); + return fieldHighlighter.highlightFieldForDoc( + leaf.reader(), docId - leaf.docBase, content.toString()); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java index 249a3a9e1d7..74c658de37a 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java @@ -118,5 +118,6 @@ public abstract class TransformerFactory implements NamedListInitializedPlugin { defaultFactories.put("xml", new RawValueTransformerFactory("xml")); defaultFactories.put("geo", new GeoTransformerFactory()); defaultFactories.put("core", new CoreAugmenterFactory()); + defaultFactories.put("hl", new HighlightAugmenterFactory()); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java index cdbab82b654..3e5d0d7a0f7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java @@ -203,7 +203,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { // and a specific Jira for fixing this is listed as a comment final List<String> knownBugs = Arrays.asList( - "child" // way too complicated to check with this test, see SOLR-9379 instead + "child", // way too complicated to check with this test, see SOLR-9379 instead + "hl" // presently can only highlight via reading data from the index. ); for (String buggy : knownBugs) { diff --git a/solr/core/src/test/org/apache/solr/response/transform/TestHighlightAugmenterFactory.java b/solr/core/src/test/org/apache/solr/response/transform/TestHighlightAugmenterFactory.java new file mode 100644 index 00000000000..fc49fa06f0d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/response/transform/TestHighlightAugmenterFactory.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response.transform; + +import java.util.List; +import org.apache.solr.SolrTestCase; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.util.EmbeddedSolrServerTestRule; +import org.apache.solr.util.ExternalPaths; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +public class TestHighlightAugmenterFactory extends SolrTestCase { + + private static final String BODY_1 = + "The quick brown fox jumps over the lazy dog. Nothing else here. The fox ran away."; + private static final String BODY_2 = "A story about cats. Cats sleep a lot."; + + @ClassRule + public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule(); + + @BeforeClass + public static void beforeClass() throws Exception { + solrTestRule.startSolr(); + solrTestRule.newCollection().withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); + + SolrClient client = solrTestRule.getSolrClient(); + SolrInputDocument doc1 = new SolrInputDocument(); + doc1.addField("id", "1"); + doc1.addField("body_t", BODY_1); + doc1.addField("title_t", "Fox story"); + doc1.addField("tags_txt", List.of("animals", "forest fox")); + client.add(doc1); + SolrInputDocument doc2 = new SolrInputDocument(); + doc2.addField("id", "2"); + doc2.addField("body_t", BODY_2); + doc2.addField("title_t", "Fox cameo"); + doc2.addField("tags_txt", List.of("pets", "cats")); + client.add(doc2); + client.commit(); + } + + /** Queries for "fox" in body_t or title_t, sorted by id; doc 2 has no "fox" in its body. */ + private static List<SolrDocument> query(String fl, String... extraParams) throws Exception { + SolrQuery query = new SolrQuery("body_t:fox title_t:fox"); + query.setFields(fl); + query.set("sort", "id asc"); + for (int i = 0; i < extraParams.length; i += 2) { + query.set(extraParams[i], extraParams[i + 1]); + } + List<SolrDocument> docs = solrTestRule.getSolrClient().query(query).getResults(); + assertEquals(2, docs.size()); + return docs; + } + + @SuppressWarnings("unchecked") + private static List<String> snippets(SolrDocument doc, String name) { + return (List<String>) doc.getFieldValue(name); + } + + private static void assertContains(String expected, Object actual) { + assertTrue(String.valueOf(actual), actual instanceof String s && s.contains(expected)); + } + + @Test + public void testExplicitField() throws Exception { + // single-valued field and 1 snippet: a string, not a list + List<SolrDocument> docs = query("id,body_hl:[hl f=body_t]"); + assertContains("<em>fox</em>", docs.getFirst().getFieldValue("body_hl")); + assertNull("the highlighted field isn't returned", docs.getFirst().getFieldValue("body_t")); + // no highlight, and the name isn't the field, so no pass-through + assertFalse(docs.get(1).containsKey("body_hl")); + + // a list when multiple snippets may be returned + docs = query("id,body_hl:[hl f=body_t snippets=2]"); + assertEquals(List.of(), docs.get(1).getFieldValue("body_hl")); + } + + @Test + public void testFieldFromAlias() throws Exception { + List<SolrDocument> docs = query("id,body_t:[hl]"); + assertContains("<em>fox</em>", docs.getFirst().getFieldValue("body_t")); + // no highlight: the original value passes through + assertEquals(BODY_2, docs.get(1).getFieldValue("body_t")); + + // also when the name equals an explicitly given field + docs = query("id,body_t:[hl f=body_t]"); + assertEquals(BODY_2, docs.get(1).getFieldValue("body_t")); + } + + @Test + public void testMultiValuedPassThrough() throws Exception { + List<SolrDocument> docs = query("id,tags_txt:[hl]"); + List<String> snippets = snippets(docs.getFirst(), "tags_txt"); + assertEquals(List.of("forest <em>fox</em>"), snippets); + assertEquals(List.of("pets", "cats"), docs.get(1).getFieldValue("tags_txt")); + } + + @Test + public void testLocalParams() throws Exception { + List<SolrDocument> docs = + query("id,body_hl:[hl f=body_t snippets=2 fragsize=1 tag.pre='[' tag.post=']']"); + List<String> snippets = snippets(docs.getFirst(), "body_hl"); + assertEquals(snippets.toString(), 2, snippets.size()); + for (String snippet : snippets) { + assertTrue(snippet, snippet.contains("[fox]")); + } + + // q overrides the main query + docs = query("id,body_hl:[hl f=body_t q=body_t:cats]"); + assertFalse(docs.getFirst().containsKey("body_hl")); + assertContains("<em>cats</em>", docs.get(1).getFieldValue("body_hl")); + + // defaultSummary returns leading text when nothing matches + docs = query("id,body_hl:[hl f=body_t defaultSummary=true fragsize=0]"); + assertEquals(BODY_2, docs.get(1).getFieldValue("body_hl")); + } + + @Test + public void testRequestParamFallback() throws Exception { + List<SolrDocument> docs = + query("id,body_hl:[hl f=body_t]", "hl.tag.pre", "[", "hl.tag.post", "]"); + assertContains("[fox]", docs.getFirst().getFieldValue("body_hl")); + + // local params override the request's, even per-field ones + docs = query("id,body_hl:[hl f=body_t tag.pre=< tag.post=>]", "hl.tag.pre", "["); + assertContains("<fox>", docs.getFirst().getFieldValue("body_hl")); + docs = query("id,body_hl:[hl f=body_t tag.pre=< tag.post=>]", "f.body_t.hl.tag.pre", "["); + assertContains("<fox>", docs.getFirst().getFieldValue("body_hl")); + + // the request's snippets param determines if the output is a list + docs = query("id,body_hl:[hl f=body_t]", "hl.snippets", "2"); + assertEquals(List.of(), docs.get(1).getFieldValue("body_hl")); + } + + @Test + public void testMultipleInstances() throws Exception { + List<SolrDocument> docs = query("id,body_t:[hl],title_t:[hl tag.pre=* tag.post=*]"); + assertContains("<em>fox</em>", docs.getFirst().getFieldValue("body_t")); + assertEquals("*Fox* story", docs.getFirst().getFieldValue("title_t")); + } + + private static ModifiableSolrParams boostedQuery(boolean weightMatches) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("q", "fox"); + params.set("defType", "edismax"); + params.set("qf", "body_t title_t"); + params.set("bq", "body_t:cats"); + params.set("fl", "id,body_hl:[hl f=body_t]"); + params.set("sort", "id asc"); + params.set("hl.weightMatches", weightMatches); + params.set("hl.defaultSummary", false); + params.set("hl.method", "unified"); + params.set("hl.fl", "body_t"); + return params; + } + + private static void assertOnlyUserQueryHighlighted(QueryResponse response) { + List<SolrDocument> docs = response.getResults(); + assertEquals(2, docs.size()); + assertEquals("1", docs.getFirst().getFieldValue("id")); + assertEquals("2", docs.get(1).getFieldValue("id")); + assertContains("<em>fox</em>", docs.getFirst().getFieldValue("body_hl")); + assertFalse( + "Boost-only text must not be highlighted: " + docs.get(1), + docs.get(1).containsKey("body_hl")); + if (response.getHighlighting() != null) { + var highlights = response.getHighlighting().get("2"); + assertTrue(String.valueOf(highlights), highlights == null || highlights.isEmpty()); + } + } + + @Test + public void testImplicitQueryUsesHighlightQuery() throws Exception { + for (boolean weightMatches : List.of(true, false)) { + for (boolean component : List.of(false, true)) { + ModifiableSolrParams params = boostedQuery(weightMatches); + params.set("hl", component); + assertOnlyUserQueryHighlighted(solrTestRule.getSolrClient().query(params)); + + params.set("hl.q", "fox"); + assertOnlyUserQueryHighlighted(solrTestRule.getSolrClient().query(params)); + + params.remove("hl.q"); + params.set("fl", "id,body_hl:[hl f=body_t q=fox]"); + assertOnlyUserQueryHighlighted(solrTestRule.getSolrClient().query(params)); + } + } + } + + @Test + public void testImplicitQueryUsesMainParser() throws Exception { + ModifiableSolrParams params = boostedQuery(false); + params.remove("q"); + params.set("q.alt", "{!lucene}body_t:fox OR title_t:fox"); + assertOnlyUserQueryHighlighted(solrTestRule.getSolrClient().query(params)); + + params = boostedQuery(false); + params.remove("defType"); + params.remove("qf"); + params.remove("bq"); + params.set("q", "{!edismax qf='body_t title_t' bq='body_t:cats'}fox"); + assertOnlyUserQueryHighlighted(solrTestRule.getSolrClient().query(params)); + } + + @Test + public void testExplicitHighlightQueryOverridesMainQuery() throws Exception { + for (boolean weightMatches : List.of(true, false)) { + ModifiableSolrParams params = boostedQuery(weightMatches); + params.set("hl.q", "body_t:cats"); + List<SolrDocument> docs = solrTestRule.getSolrClient().query(params).getResults(); + assertEquals(2, docs.size()); + assertFalse(docs.getFirst().containsKey("body_hl")); + assertContains("<em>cats</em>", docs.get(1).getFieldValue("body_hl")); + + params.set("hl.q", "fox"); + params.set("fl", "id,body_hl:[hl f=body_t q=body_t:cats]"); + docs = solrTestRule.getSolrClient().query(params).getResults(); + assertEquals(2, docs.size()); + assertFalse(docs.getFirst().containsKey("body_hl")); + assertContains("<em>cats</em>", docs.get(1).getFieldValue("body_hl")); + } + } + + @Test + public void testRealTimeGet() throws Exception { + // no query, so the original value passes through + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("fl", "id,body_t:[hl]"); + SolrDocument doc = solrTestRule.getSolrClient().getById("1", params); + assertEquals(BODY_1, doc.getFieldValue("body_t")); + } + + @Test + public void testErrors() { + SolrException e = expectThrows(SolrException.class, () -> query("id,[hl]")); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + e = expectThrows(SolrException.class, () -> query("id,body_t:[hl bogus=1]")); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code()); + } +} diff --git a/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc index e454ebafd97..292ec43f0c4 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc @@ -411,6 +411,41 @@ As such, you needn't mark the field as stored, which would be redundant. In a sense this double-storage between docValues and stored-value storage isn't unique to spatial but with polygonal geometry it can be a lot of data, and furthermore you'd like to avoid storing it in a verbose format (like GeoJSON or WKT). +=== [hl] - HighlightAugmenterFactory + +Highlights one field of each document using the xref:highlighting.adoc#unified-highlighter[Unified Highlighter], and adds the snippets to the document as a list. +If the field is single-valued and one snippet is requested (the default), the snippet is added as a single value instead of a list. +Unlike the highlight component, the snippets are inline in each document instead of in a separate `highlighting` section of the response. + +The field to highlight is given by the `f` parameter; if omitted, it's the transformer's name (its alias). +For example, `fl=id,body_hl:[hl f=body]` adds `body_hl` with snippets of `body`, without returning the full `body`. +And `fl=id,body:[hl]` returns snippets of `body` as `body`. +List multiple instances to highlight multiple fields: `fl=id,body:[hl],title:[hl snippets=3]`. + +[source,plain] +---- +q=body:fox&fl=id,body:[hl] +---- + +[source,json] +---- +{ "response":{"numFound":1,"start":0,"docs":[ + { + "id":"1", + "body":"The quick brown <em>fox</em> jumps over the lazy dog."}] +}} +---- + +If nothing in the field is highlighted, the result depends on the name: + +* If the name is the field (e.g., `body:[hl]`), the field's original value is returned, so the field is returned either highlighted or as-is. +* Otherwise (e.g., `body_hl:[hl f=body]`), the result is an empty list, or absent when a single value would have been returned. +* With `defaultSummary=true`, the leading text of the field is returned as a snippet instead. + +The highlighting is configured by the request's xref:highlighting.adoc[highlighting parameters], such as `hl.tag.pre`; `hl=true` is not needed. +They can be overridden for one transformer, including any per-field form like `f.body.hl.snippets`, by giving them without the `hl.` prefix, e.g. `[hl f=body snippets=3 tag.pre='<b>' tag.post='</b>']`, or `q` to highlight a query other than the main one. + + === [features] - LTRFeatureLoggerTransformerFactory The "LTR" prefix stands for xref:learning-to-rank.adoc[]. diff --git a/solr/solr-ref-guide/modules/query-guide/pages/highlighting.adoc b/solr/solr-ref-guide/modules/query-guide/pages/highlighting.adoc index 5b51f940050..c98fa6daeeb 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/highlighting.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/highlighting.adoc @@ -17,9 +17,19 @@ // under the License. Highlighting in Solr allows fragments of documents that match the user's query to be included with the query response. - -The fragments are included in a special section of the query response (the `highlighting` section), and the client uses the formatting clues also included to determine how to present the snippets to users. Fragments are a portion of a document field that contains matches from the query and are sometimes also referred to as "snippets" or "passages". +The client uses the formatting clues also included to determine how to present the snippets to users. + +There are two ways to request highlighting: + +* The highlight component, enabled with `hl=true`, includes the fragments in a special section of the query response (the `highlighting` section), separate from the documents. +This is what most of this page describes. +* The xref:document-transformers.adoc#hl-highlightaugmenterfactory[`[hl\]` document transformer], requested in `fl`, includes the fragments in each document, e.g., `fl=id,body:[hl]`. +It always uses the <<Unified Highlighter>>, configured with the same parameters. + +Choosing between them is mostly a matter of preference, though there are implications at scale. +The transformer is preferred when highlighting hundreds of documents or more, since each document is highlighted as the results are streamed. +On the other hand, the transformer has more overhead per field, which could be a concern when highlighting many fields. Highlighting is extremely configurable, perhaps more than any other part of Solr. There are many parameters each for fragment sizing, formatting, ordering, backup/alternate behavior, and more options that are hard to categorize.
