Croway commented on code in PR #27814:
URL: https://github.com/apache/camel/pull/27814#discussion_r4269314210
##########
dsl/camel-jbang/camel-jbang-mcp/src/test/java/org/apache/camel/dsl/jbang/core/commands/mcp/MigrationDataSearchTest.java:
##########
@@ -55,6 +56,82 @@ void nonsenseQueryReturnsNoResults() {
assertThat(results).isEmpty();
}
+ @Test
+ void unrelatedOrStopWordsOnlyQueryReturnsNoResults() {
+ // "smoothie" is within typo distance of "smoothing", but an entry
matching a third of the query is unrelated
+ assertThat(migrationData.searchGuides("banana smoothie recipe",
3)).isEmpty();
+ assertThat(migrationData.searchGuides("camel migration upgrade guide",
3)).isEmpty();
+ }
+
+ @Test
+ void entriesAreTopicsWithTheReleaseHeadingAsContext() {
+ assertThat(migrationData.getGuideIndex())
+ .anyMatch(s -> s.sectionTitle().equals("Upgrading Camel 4.20
to 4.21 / camel-cxf"))
+ .allMatch(s -> s.content().lines().noneMatch(l ->
l.startsWith("== ") || l.startsWith("=== ")));
+ }
+
+ @Test
+ void cxfServletFindsCamelCxfEntry() {
+ List<MigrationData.GuideSection> results =
migrationData.searchGuides("cxf servlet", 3);
+
+ assertThat(results).isNotEmpty();
+ assertThat(results.get(0).sectionTitle()).endsWith("/ camel-cxf");
+ assertThat(results.get(0).content()).contains("camel-servlet");
+ }
+
+ @Test
+ void camelMailSubjectHeaderFindsCamelMailEntry() {
+ List<MigrationData.GuideSection> results =
migrationData.searchGuides("camel-mail subject header", 3);
+
+ assertThat(results).isNotEmpty();
+ assertThat(results.get(0).sectionTitle()).contains("/ camel-mail");
+ assertThat(results.get(0).content()).contains("MAIL_SUBJECT");
+ }
+
+ @Test
+ void versionRangeRanksMajorMigrationGuidesFirst() {
+ List<MigrationData.SearchHit> hits
+ = migrationData.search("CXF SOAP web service migration Camel 2
to 4", "2.25", "4.22", 5);
+
+ assertThat(hits).isNotEmpty();
+ assertThat(hits.get(0).section().guide()).isEqualTo("Camel 3 Migration
Guide (2.x to 3.0)");
+ assertThat(hits.get(0).section().sectionTitle()).startsWith("Migrating
Camel applications");
+ assertThat(hits.get(0).section().content()).contains("camel-cxf");
+ }
+
+ @Test
+ void versionRangeOmitsGuidesOutsideTheRange() {
+ List<MigrationData.SearchHit> hits = migrationData.search("cxf
servlet", "4.10", "4.22", 20);
+
+ assertThat(hits).isNotEmpty();
+ assertThat(hits).extracting(h -> h.section().version())
+ .allMatch(v -> v.startsWith("4.") &&
Integer.parseInt(v.substring(2)) > 10
+ && Integer.parseInt(v.substring(2)) <= 22);
+ }
+
+ @Test
+ void invalidVersionIsRejected() {
+ assertThatThrownBy(() -> migrationData.search("cxf", "four", null, 3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("fromVersion");
+ assertThatThrownBy(() -> migrationData.search("cxf", "4.22", "3.20",
3))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("is after toVersion");
+ }
+
+ @Test
+ void snippetShowsTheLinesAroundTheMatch() {
+ List<MigrationData.SearchHit> hits = migrationData.search("camel-mail
subject header", null, null, 1);
+
+ assertThat(hits).hasSize(1);
+ String content = hits.get(0).section().content();
+ String snippet = migrationData.snippet(hits.get(0), 10);
+
+ assertThat(content.lines().count()).isGreaterThan(10);
+ assertThat(content.lines().limit(10)).noneMatch(l ->
l.contains("MAIL_SUBJECT"));
+ assertThat(snippet).contains("MAIL_SUBJECT").contains("lines above");
+ }
+
@Test
void searchRespectsLimit() {
List<MigrationData.GuideSection> results =
migrationData.searchGuides("camel", 2);
Review Comment:
Done: the test searches "cxf" with limit 2 and asserts `hasSize(2)`, so it
fails if the limit is ignored.
##########
dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/MigrationTools.java:
##########
@@ -289,53 +290,61 @@ public MigrationRecipesResult camel_migration_recipes(
}
/**
- * Search migration guides for a specific term.
+ * Search migration guides for a specific term, optionally scoped to the
versions being migrated between.
*/
@Tool(annotations = @Tool.Annotations(readOnlyHint = true, destructiveHint
= false, openWorldHint = false),
description = "Search Camel migration and upgrade guides for a
specific term or component name. "
- + "Returns matching snippets from the official guides
with version info and URLs. "
- + "Supports fuzzy matching for typo tolerance. "
+ + "Each result is one topic entry of a release (e.g.
'Upgrading Camel 4.20 to 4.21 / camel-cxf') "
+ + "with the lines around the best match, version info
and URL. "
+ + "Pass fromVersion/toVersion (e.g. 2.25 and 4.22) to
search only the guides for that "
+ + "migration, with the 2.x to 3.0 and 3.x to 4.0
migration guides ranked first. "
+ + "Results are ranked by relevance with typo
tolerance; generic words such as camel, "
+ + "migration or upgrade are ignored, and no results
are returned when nothing is relevant. "
+ "Use this instead of web search when looking up
migration-related changes, "
+ "removed components, API renames, or breaking
changes.")
public GuideSearchResult camel_migration_guide_search(
@ToolArg(description = "Search query — component name, API class,
method, or keyword "
- + "(e.g., direct-vm, getOut, camel-http4,
ExchangePattern)") String query,
- @ToolArg(description = "Maximum number of results to return
(default: 3)", required = false) Integer limit) {
+ + "(e.g., direct-vm, getOut, camel-http4,
cxf servlet, camel-mail subject header)") String query,
+ @ToolArg(description = "Maximum number of results to return
(default: 3)", required = false) Integer limit,
+ @ToolArg(description = "Camel version migrating from (e.g., 2.25
or 3.20.1). "
+ + "Guides for releases up to this version
are omitted.",
+ required = false) String fromVersion,
Review Comment:
Renamed to `currentVersion` / `targetVersion` like
`camel_migration_compatibility` and `camel_migration_recipes`: tool,
description, hint, doc row and the migration prompt (f30cc1db87fe).
##########
dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/MigrationData.java:
##########
@@ -283,87 +290,310 @@ private static String getElementText(Element parent,
String tagName) {
private static final String GUIDES_RESOURCE_DIR = "migration-guides/";
private static final String GUIDES_INDEX_FILE = GUIDES_RESOURCE_DIR +
"index.txt";
- private volatile List<GuideSection> guideIndex;
+ /**
+ * Words dropped from search queries: they appear in nearly every
migration guide entry (camel, migration, upgrade,
+ * version, guide) or carry no meaning on their own (to, from, the, ...),
so they only add noise to the ranking.
+ */
+ static final Set<String> STOP_WORDS = Set.of(
+ "camel", "apache", "migration", "migrate", "migrating", "upgrade",
"upgrading", "upgraded", "version",
+ "versions", "guide", "guides", "release", "change", "changes",
"changed",
+ "a", "an", "and", "are", "as", "at", "be", "by", "do", "does",
"for", "from", "how", "in", "is", "it", "my",
+ "of", "on", "or", "the", "this", "to", "what", "when", "which",
"with");
+
+ /** BM25 term frequency saturation. */
+ private static final double BM25_K1 = 1.2;
+ /** BM25 length normalisation: 0 ignores the entry length, 1 fully
normalises by it. */
+ private static final double BM25_B = 0.5;
+ /** Extra weight, in units of the term idf, when a query term appears in
the entry heading. */
+ private static final double TITLE_BOOST = 1.5;
+ /** Weight of a part of an unknown hyphenated query term, e.g. http4 for
camel-http4. */
+ private static final double PART_WEIGHT = 0.7;
+ /** Weight of a vocabulary term matched by Levenshtein distance instead of
exactly (typo tolerance). */
+ private static final double FUZZY_WEIGHT = 0.7;
+ /** Weight of a vocabulary term that only contains the query term as a
substring. */
+ private static final double SUBSTRING_WEIGHT = 0.3;
+ /** Multiplier when the whole query appears verbatim in the entry. */
+ private static final double PHRASE_BOOST = 1.25;
+ /** Entries scoring below this are considered unrelated to the query and
are not returned. */
+ static final double MIN_SCORE = 2.5;
+
+ private volatile GuideIndex guideIndex;
/**
- * Get the guide index, loading lazily on first access.
+ * Get the guide index entries (one per {@code ===} topic of each release
section), loading lazily on first access.
*/
- public synchronized List<GuideSection> getGuideIndex() {
+ public List<GuideSection> getGuideIndex() {
+ return index().sections();
+ }
+
+ private synchronized GuideIndex index() {
if (guideIndex == null) {
- guideIndex = loadGuideSections();
+ guideIndex = buildIndex(loadGuideSections());
}
return guideIndex;
}
/**
- * Search migration guides for sections matching the query using fuzzy
matching.
- * <p>
- * Scoring: +10 if the full query appears verbatim in the section content.
Per query token: +3 exact token match, +2
- * fuzzy match (Levenshtein distance ≤ max(1, minTokenLen/4)), +1
substring fallback. Results are sorted by
- * descending score.
+ * Search all migration guides, see {@link #search(String, String, String,
int)}.
*/
public List<GuideSection> searchGuides(String query, int maxResults) {
- List<String> queryTokens = tokenize(query);
- String lowerQuery = query.toLowerCase(Locale.ROOT);
+ return search(query, null, null,
maxResults).stream().map(SearchHit::section).toList();
+ }
- List<GuideSection> index = getGuideIndex();
- List<ScoredSection> scored = new ArrayList<>();
+ /**
+ * Search the migration guides for the entries matching the query.
+ * <p>
+ * Each entry is one {@code ===} topic (usually one component) of one
release section of a guide. Entries are ranked
+ * with BM25 over the query terms (stop words such as camel, migration or
upgrade removed), with a bonus for terms
+ * in the entry heading. A query term that does not occur verbatim also
matches vocabulary terms within a small
+ * Levenshtein distance (typos) or containing it, at a reduced weight.
Entries matching less than half of the query
+ * terms or scoring below {@link #MIN_SCORE} are dropped, so an unrelated
query returns no results.
+ * <p>
+ * When {@code fromVersion} and/or {@code toVersion} are given, only the
guides for the releases after
+ * {@code fromVersion} up to {@code toVersion} are searched, and the
entries of the major migration guides (2.x to
+ * 3.0, 3.x to 4.0) in that range are listed before the entries of the
minor upgrade guides.
+ *
+ * @param query the search query
+ * @param fromVersion the Camel version migrated from (e.g.
2.25 or 3.20.1), or null
+ * @param toVersion the Camel version migrated to (e.g.
4.22), or null
+ * @param maxResults the maximum number of results
+ * @return the matching entries, best first
+ * @throws IllegalArgumentException if a version cannot be parsed or
fromVersion is after toVersion
+ */
+ public List<SearchHit> search(String query, String fromVersion, String
toVersion, int maxResults) {
+ int[] from = parseVersion(fromVersion, "fromVersion");
+ int[] to = parseVersion(toVersion, "toVersion");
+ if (from != null && to != null && compareMajorMinor(from, to) > 0) {
+ throw new IllegalArgumentException(
+ "fromVersion " + fromVersion + " is after toVersion " +
toVersion);
+ }
+ boolean ranged = from != null || to != null;
- for (GuideSection section : index) {
- double score = scoreSection(section, queryTokens, lowerQuery);
- if (score > 0) {
- scored.add(new ScoredSection(section, score));
- }
+ List<String> queryTerms = queryTerms(query);
+ if (queryTerms.isEmpty() || maxResults <= 0) {
+ return List.of();
}
-
scored.sort(Comparator.comparingDouble(ScoredSection::score).reversed());
+ GuideIndex index = index();
+ List<Map<String, Double>> expansions = new ArrayList<>();
+ for (String term : queryTerms) {
+ expansions.add(expandTerm(term, index));
+ }
+ String phrase = query.toLowerCase(Locale.ROOT).trim();
- return scored.stream()
- .limit(maxResults)
- .map(ScoredSection::section)
- .collect(Collectors.toList());
+ List<SearchHit> hits = new ArrayList<>();
+ for (GuideSection section : index.sections()) {
+ if (ranged && !inRange(section, from, to)) {
+ continue;
+ }
+ Set<String> matched = new LinkedHashSet<>();
+ double score = 0;
+ int matchedTerms = 0;
+ for (Map<String, Double> expansion : expansions) {
+ double best = 0;
+ String bestTerm = null;
+ for (Map.Entry<String, Double> variant : expansion.entrySet())
{
+ double s = scoreTerm(variant.getKey(), section, index) *
variant.getValue();
+ if (s > best) {
+ best = s;
+ bestTerm = variant.getKey();
+ }
+ }
+ if (bestTerm != null) {
+ matched.add(bestTerm);
+ matchedTerms++;
+ score += best;
+ }
+ }
+ if (matchedTerms == 0 || matchedTerms * 2 < expansions.size()) {
+ // an entry matching less than half of the query is about
something else
+ continue;
+ }
+ // favour entries covering more of the query
+ score *= 0.5 + 0.5 * matchedTerms / expansions.size();
+ if (queryTerms.size() > 1 &&
section.content().toLowerCase(Locale.ROOT).contains(phrase)) {
+ score *= PHRASE_BOOST;
+ }
+ if (score >= MIN_SCORE) {
+ hits.add(new SearchHit(section, score, List.copyOf(matched)));
+ }
+ }
+
+ Comparator<SearchHit> byScore =
Comparator.comparingDouble(SearchHit::score).reversed();
+ if (ranged) {
+ // migrating across a major version: the 2.x to 3.0 and 3.x to 4.0
migration guides come first
+ hits.sort(Comparator.comparing((SearchHit h) ->
!h.section().majorMigration()).thenComparing(byScore));
Review Comment:
Good catch, and "cxf servlet" with 2.25/4.22 is now a test. The partition is
replaced by a score multiplier for the major-guide entries. The factor is 1.85,
not 1.5: for the third JIRA query the 2.x-to-3.0 "Migrating Camel applications"
entry scores 9.41 against 17.03 for the 4.21 camel-cxf note, so the break-even
is 1.81. That margin is thin, and a later edit of the 4.21 note could flip that
order; if you would rather drop the "major guides first" expectation and keep a
plain 1.5, say so and I will.
##########
dsl/camel-jbang/camel-jbang-mcp/src/main/java/org/apache/camel/dsl/jbang/core/commands/mcp/MigrationData.java:
##########
@@ -283,87 +290,310 @@ private static String getElementText(Element parent,
String tagName) {
private static final String GUIDES_RESOURCE_DIR = "migration-guides/";
private static final String GUIDES_INDEX_FILE = GUIDES_RESOURCE_DIR +
"index.txt";
- private volatile List<GuideSection> guideIndex;
+ /**
+ * Words dropped from search queries: they appear in nearly every
migration guide entry (camel, migration, upgrade,
+ * version, guide) or carry no meaning on their own (to, from, the, ...),
so they only add noise to the ranking.
+ */
+ static final Set<String> STOP_WORDS = Set.of(
+ "camel", "apache", "migration", "migrate", "migrating", "upgrade",
"upgrading", "upgraded", "version",
+ "versions", "guide", "guides", "release", "change", "changes",
"changed",
+ "a", "an", "and", "are", "as", "at", "be", "by", "do", "does",
"for", "from", "how", "in", "is", "it", "my",
+ "of", "on", "or", "the", "this", "to", "what", "when", "which",
"with");
+
+ /** BM25 term frequency saturation. */
+ private static final double BM25_K1 = 1.2;
+ /** BM25 length normalisation: 0 ignores the entry length, 1 fully
normalises by it. */
+ private static final double BM25_B = 0.5;
+ /** Extra weight, in units of the term idf, when a query term appears in
the entry heading. */
+ private static final double TITLE_BOOST = 1.5;
+ /** Weight of a part of an unknown hyphenated query term, e.g. http4 for
camel-http4. */
+ private static final double PART_WEIGHT = 0.7;
+ /** Weight of a vocabulary term matched by Levenshtein distance instead of
exactly (typo tolerance). */
+ private static final double FUZZY_WEIGHT = 0.7;
+ /** Weight of a vocabulary term that only contains the query term as a
substring. */
+ private static final double SUBSTRING_WEIGHT = 0.3;
+ /** Multiplier when the whole query appears verbatim in the entry. */
+ private static final double PHRASE_BOOST = 1.25;
+ /** Entries scoring below this are considered unrelated to the query and
are not returned. */
+ static final double MIN_SCORE = 2.5;
+
+ private volatile GuideIndex guideIndex;
/**
- * Get the guide index, loading lazily on first access.
+ * Get the guide index entries (one per {@code ===} topic of each release
section), loading lazily on first access.
*/
- public synchronized List<GuideSection> getGuideIndex() {
+ public List<GuideSection> getGuideIndex() {
+ return index().sections();
+ }
+
+ private synchronized GuideIndex index() {
if (guideIndex == null) {
- guideIndex = loadGuideSections();
+ guideIndex = buildIndex(loadGuideSections());
}
return guideIndex;
}
/**
- * Search migration guides for sections matching the query using fuzzy
matching.
- * <p>
- * Scoring: +10 if the full query appears verbatim in the section content.
Per query token: +3 exact token match, +2
- * fuzzy match (Levenshtein distance ≤ max(1, minTokenLen/4)), +1
substring fallback. Results are sorted by
- * descending score.
+ * Search all migration guides, see {@link #search(String, String, String,
int)}.
*/
public List<GuideSection> searchGuides(String query, int maxResults) {
- List<String> queryTokens = tokenize(query);
- String lowerQuery = query.toLowerCase(Locale.ROOT);
+ return search(query, null, null,
maxResults).stream().map(SearchHit::section).toList();
+ }
- List<GuideSection> index = getGuideIndex();
- List<ScoredSection> scored = new ArrayList<>();
+ /**
+ * Search the migration guides for the entries matching the query.
+ * <p>
+ * Each entry is one {@code ===} topic (usually one component) of one
release section of a guide. Entries are ranked
+ * with BM25 over the query terms (stop words such as camel, migration or
upgrade removed), with a bonus for terms
+ * in the entry heading. A query term that does not occur verbatim also
matches vocabulary terms within a small
+ * Levenshtein distance (typos) or containing it, at a reduced weight.
Entries matching less than half of the query
+ * terms or scoring below {@link #MIN_SCORE} are dropped, so an unrelated
query returns no results.
+ * <p>
+ * When {@code fromVersion} and/or {@code toVersion} are given, only the
guides for the releases after
+ * {@code fromVersion} up to {@code toVersion} are searched, and the
entries of the major migration guides (2.x to
+ * 3.0, 3.x to 4.0) in that range are listed before the entries of the
minor upgrade guides.
+ *
+ * @param query the search query
+ * @param fromVersion the Camel version migrated from (e.g.
2.25 or 3.20.1), or null
+ * @param toVersion the Camel version migrated to (e.g.
4.22), or null
+ * @param maxResults the maximum number of results
+ * @return the matching entries, best first
+ * @throws IllegalArgumentException if a version cannot be parsed or
fromVersion is after toVersion
+ */
+ public List<SearchHit> search(String query, String fromVersion, String
toVersion, int maxResults) {
+ int[] from = parseVersion(fromVersion, "fromVersion");
+ int[] to = parseVersion(toVersion, "toVersion");
+ if (from != null && to != null && compareMajorMinor(from, to) > 0) {
+ throw new IllegalArgumentException(
+ "fromVersion " + fromVersion + " is after toVersion " +
toVersion);
+ }
+ boolean ranged = from != null || to != null;
- for (GuideSection section : index) {
- double score = scoreSection(section, queryTokens, lowerQuery);
- if (score > 0) {
- scored.add(new ScoredSection(section, score));
- }
+ List<String> queryTerms = queryTerms(query);
+ if (queryTerms.isEmpty() || maxResults <= 0) {
+ return List.of();
}
-
scored.sort(Comparator.comparingDouble(ScoredSection::score).reversed());
+ GuideIndex index = index();
+ List<Map<String, Double>> expansions = new ArrayList<>();
+ for (String term : queryTerms) {
+ expansions.add(expandTerm(term, index));
+ }
+ String phrase = query.toLowerCase(Locale.ROOT).trim();
- return scored.stream()
- .limit(maxResults)
- .map(ScoredSection::section)
- .collect(Collectors.toList());
+ List<SearchHit> hits = new ArrayList<>();
+ for (GuideSection section : index.sections()) {
+ if (ranged && !inRange(section, from, to)) {
+ continue;
+ }
+ Set<String> matched = new LinkedHashSet<>();
+ double score = 0;
+ int matchedTerms = 0;
+ for (Map<String, Double> expansion : expansions) {
+ double best = 0;
+ String bestTerm = null;
+ for (Map.Entry<String, Double> variant : expansion.entrySet())
{
+ double s = scoreTerm(variant.getKey(), section, index) *
variant.getValue();
+ if (s > best) {
+ best = s;
+ bestTerm = variant.getKey();
+ }
+ }
+ if (bestTerm != null) {
+ matched.add(bestTerm);
+ matchedTerms++;
+ score += best;
+ }
+ }
+ if (matchedTerms == 0 || matchedTerms * 2 < expansions.size()) {
+ // an entry matching less than half of the query is about
something else
+ continue;
+ }
+ // favour entries covering more of the query
+ score *= 0.5 + 0.5 * matchedTerms / expansions.size();
+ if (queryTerms.size() > 1 &&
section.content().toLowerCase(Locale.ROOT).contains(phrase)) {
+ score *= PHRASE_BOOST;
+ }
+ if (score >= MIN_SCORE) {
+ hits.add(new SearchHit(section, score, List.copyOf(matched)));
+ }
+ }
+
+ Comparator<SearchHit> byScore =
Comparator.comparingDouble(SearchHit::score).reversed();
+ if (ranged) {
+ // migrating across a major version: the 2.x to 3.0 and 3.x to 4.0
migration guides come first
+ hits.sort(Comparator.comparing((SearchHit h) ->
!h.section().majorMigration()).thenComparing(byScore));
+ } else {
+ hits.sort(byScore);
+ }
+ return hits.size() > maxResults ? List.copyOf(hits.subList(0,
maxResults)) : hits;
}
- private double scoreSection(GuideSection section, List<String>
queryTokens, String lowerQuery) {
- double score = 0;
- String lowerContent = section.content().toLowerCase(Locale.ROOT);
+ /**
+ * Return the lines of the entry around its best match: the line
containing the most (idf weighted) matched terms,
+ * with some lines of context before it, and at most {@code maxLines}
lines.
+ */
+ public String snippet(SearchHit hit, int maxLines) {
+ String[] lines = hit.section().content().split("\n");
+ if (lines.length <= maxLines) {
+ return hit.section().content();
+ }
+ GuideIndex index = index();
+ int bestLine = 0;
+ double bestScore = 0;
+ for (int i = 0; i < lines.length; i++) {
+ Set<String> lineTerms = new HashSet<>(tokenize(lines[i]));
+ double s = 0;
+ for (String term : hit.matchedTerms()) {
+ if (lineTerms.contains(term)) {
+ s += idf(term, index);
+ }
+ }
+ if (s > bestScore) {
+ bestScore = s;
+ bestLine = i;
+ }
+ }
+ int start = Math.max(0, bestLine - maxLines / 4);
+ int end = Math.min(lines.length, start + maxLines);
+ start = Math.max(0, end - maxLines);
- // Bonus for exact substring match of the full query
- if (lowerContent.contains(lowerQuery)) {
- score += 10;
+ StringBuilder sb = new StringBuilder();
+ if (start > 0) {
+ sb.append("... (").append(start).append(" lines above)\n");
+ }
+ for (int i = start; i < end; i++) {
+ sb.append(lines[i]).append("\n");
+ }
+ if (end < lines.length) {
+ sb.append("... (truncated, ").append(lines.length - end).append("
more lines)");
+ }
+ return sb.toString();
+ }
+
+ private static double scoreTerm(String term, GuideSection section,
GuideIndex index) {
+ Integer tf = section.termFrequencies().get(term);
+ if (tf == null) {
+ return 0;
+ }
+ double idf = idf(term, index);
+ double norm = BM25_K1 * (1 - BM25_B + BM25_B * section.length() /
index.averageLength());
+ double score = idf * tf * (BM25_K1 + 1) / (tf + norm);
+ if (section.titleTerms().contains(term)) {
+ score += idf * TITLE_BOOST;
}
+ return score;
+ }
- // Token-level scoring
- for (String qt : queryTokens) {
- boolean exactFound = false;
- boolean fuzzyFound = false;
- boolean substringFound = false;
+ private static double idf(String term, GuideIndex index) {
+ int df = index.documentFrequencies().getOrDefault(term, 0);
+ int n = index.sections().size();
+ return Math.log(1 + (n - df + 0.5) / (df + 0.5));
+ }
- for (String dt : section.tokens()) {
- if (dt.equals(qt)) {
- exactFound = true;
- break;
+ /**
+ * The vocabulary terms a query term matches, with their weight: the term
itself and the terms containing it. When
+ * the term itself does not occur in the guides, also the parts of a
hyphenated term (http4 for camel-http4) or else
+ * the terms within a small Levenshtein distance (typos, e.g. directvm for
direct-vm).
+ */
+ private static Map<String, Double> expandTerm(String term, GuideIndex
index) {
+ Set<String> vocabulary = index.documentFrequencies().keySet();
+ Map<String, Double> variants = new HashMap<>();
+ if (vocabulary.contains(term)) {
+ variants.put(term, 1.0);
+ }
+ if (variants.isEmpty() && term.indexOf('-') > 0) {
+ for (String part : term.split("-+")) {
+ if (part.length() > 1 && !STOP_WORDS.contains(part) &&
vocabulary.contains(part)) {
+ variants.putIfAbsent(part, PART_WEIGHT);
}
- int maxDist = Math.max(1, Math.min(qt.length(), dt.length()) /
4);
- if (levenshteinDistance(qt, dt) <= maxDist) {
- fuzzyFound = true;
+ }
+ }
+ if (term.length() >= 4) {
+ boolean known = !variants.isEmpty();
+ for (String candidate : vocabulary) {
+ if (candidate.equals(term)) {
+ continue;
+ }
+ int maxDist = Math.max(1, Math.min(term.length(),
candidate.length()) / 4);
+ if (!known && Math.abs(candidate.length() - term.length()) <=
maxDist
+ && levenshteinDistance(term, candidate) <= maxDist) {
+ variants.putIfAbsent(candidate, FUZZY_WEIGHT);
+ } else if (candidate.contains(term)) {
+ variants.putIfAbsent(candidate, SUBSTRING_WEIGHT);
}
}
+ }
+ return variants;
+ }
- if (!exactFound && !fuzzyFound && lowerContent.contains(qt)) {
- substringFound = true;
+ /**
+ * The distinct search terms of a query: its words without stop words and
bare numbers. A hyphenated word stays one
+ * term, its parts are matched as variants of it.
+ */
+ static List<String> queryTerms(String query) {
+ if (query == null) {
+ return List.of();
+ }
+ Set<String> terms = new LinkedHashSet<>();
+ for (String token : words(query)) {
+ if (!STOP_WORDS.contains(token) &&
!token.chars().allMatch(Character::isDigit)) {
+ terms.add(token);
}
+ }
+ return new ArrayList<>(terms);
+ }
- if (exactFound) {
- score += 3;
- } else if (fuzzyFound) {
- score += 2;
- } else if (substringFound) {
- score += 1;
+ private static boolean inRange(GuideSection section, int[] from, int[] to)
{
+ int[] version = parseVersion(section.version(), "guide version");
Review Comment:
Fixed: a guide version that does not parse is skipped (`inRange` returns
false) instead of failing every ranged search.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
