This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_9x in repository https://gitbox.apache.org/repos/asf/solr.git
commit a268ba5048a23b7f8393c7a45a4cb15db63db375 Author: Renato Haeberli <[email protected]> AuthorDate: Thu Oct 22 12:38:33 2026 +0200 ensure original token is not set to type:SYNONYM in SGF (#4960) Co-authored-by: Eric Pugh <[email protected]> (cherry picked from commit 51a78ca464bfb69cdd152db0c226dcaf694c59e1) (cherry picked from commit 682551161e55fc5f8753f13435180c5b100e8f6f) --- ...nal-token-type-managed-synonym-graph-filter.yml | 9 ++++++++ .../analysis/ManagedSynonymGraphFilterFactory.java | 20 ++++++++++++----- .../TestManagedSynonymGraphFilterFactory.java | 26 ++++++++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml new file mode 100644 index 00000000000..08725631593 --- /dev/null +++ b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: ManagedSynonymGraphFilterFactory no longer tags the original token as type `SYNONYM` for equivalence-group mappings (e.g. `[terma]->[terma, termb, termc]`), using Lucene's `includeOrig` flag so only the injected synonyms get that type. +type: fixed +authors: + - name: Renato Haeberli + nick: renatoh +links: + - name: PR#4960 + url: https://github.com/apache/solr/pull/4960 diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java index 7863ed6fc3b..00633b5b602 100644 --- a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java @@ -366,16 +366,24 @@ public class ManagedSynonymGraphFilterFactory extends BaseManagedTokenFilterFact boolean ignoreCase = synonymManager.getIgnoreCase(); for (CasePreservedSynonymMappings cpsm : synonymManager.synonymMappings.values()) { for (Map.Entry<String, Set<String>> entry : cpsm.mappings.entrySet()) { + // Use includeOrig=true so the original token keeps its upstream type and only the + // injected synonyms are tagged as SYNONYM + boolean mapsToItself = entry.getValue().contains(entry.getKey()); + CharsRef casedTerm = + analyze( + synonymManager.applyCaseSetting(ignoreCase, entry.getKey()), + new CharsRefBuilder()); for (String mapping : entry.getValue()) { - // apply the case setting to match the behavior of the SynonymMap builder - CharsRef casedTerm = - analyze( - synonymManager.applyCaseSetting(ignoreCase, entry.getKey()), - new CharsRefBuilder()); CharsRef casedMapping = analyze( synonymManager.applyCaseSetting(ignoreCase, mapping), new CharsRefBuilder()); - add(casedTerm, casedMapping, false); + if (mapsToItself) { + if (!casedTerm.equals(casedMapping)) { + add(casedTerm, casedMapping, true); + } + } else { + add(casedTerm, casedMapping, false); + } } } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java index f27bdbc0d5d..e740c55c9e1 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java @@ -262,6 +262,32 @@ public class TestManagedSynonymGraphFilterFactory extends RestTestBase { assertJDelete(endpoint + "/cheerful/joyful", "/error/code==404"); } + @Test + public void testUpstreamTokenTypeIsPreserved() throws Exception { + String endpoint = "/schema/analysis/synonyms/englishgraph"; + + List<String> equivGroup = Arrays.asList("funny", "entertaining", "whimsical"); + assertJPut(endpoint, toJSONString(equivGroup), "/responseHeader/status==0"); + + assertJQ(endpoint + "/funny", "/funny==['entertaining','funny','whimsical']"); + + restTestHarness.reload(); + + String analysisUrl = "/analysis/field?analysis.fieldtype=managed_graph_en&analysis.query=funny"; + String synFilter = "//arr[contains(@name,'SynonymGraphFilter')]"; + assertQ( + analysisUrl, + "count(" + + synFilter + + "/lst[str[@name='text'][.='funny']]/str[@name='type'][.='<ALPHANUM>']) = 1", + "count(" + + synFilter + + "/lst[str[@name='text'][.='entertaining']]/str[@name='type'][.='SYNONYM']) = 1", + "count(" + + synFilter + + "/lst[str[@name='text'][.='whimsical']]/str[@name='type'][.='SYNONYM']) = 1"); + } + /** Can we add and remove stopwords with umlauts */ @Test public void testCanHandleDecodingAndEncodingForSynonyms() throws Exception {
