txwei commented on code in PR #16243: URL: https://github.com/apache/lucene/pull/16243#discussion_r3501115049
########## lucene/core/src/java/org/apache/lucene/search/similarities/StableTflSimilarity.java: ########## @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search.similarities; + +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FieldStats; +import org.apache.lucene.search.TermStats; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.UnicodeUtil; + +/** + * StableTfl Similarity. A similarity algorithm that estimates term rarity based on term length and + * document length, rather than relying on corpus-level statistics like document frequency. + * + * <p>Inspired by BM25, this similarity model replaces IDF with a synthetic function that increases + * with term length (as a proxy for rarity) and adjusts based on document length. + * + * <p>Scoring formula: + * + * <pre> + * tf = freq / (freq + k1) + * p = 1 - (1 - m * 2 ^ (-c * |t|)) ^ |d| + * tr = log(1 + (1 - p + 0.05) / (p + 0.05)) + * score = boost * tr * tf + * </pre> + * + * <p>This similarity is parameterized by: + * + * <ul> + * <li>{@code k1} - saturation parameter for term frequency + * <li>{@code c} - decay constant controlling how term length impacts rarity + * <li>{@code k3} - query-side saturation for repeated query terms (disabled by default) + * </ul> + * + * <p>The constant {@code m} is fixed internally and controls the base term-match probability. + * + * @lucene.experimental + */ +public class StableTflSimilarity extends Similarity { + + /** Default k1 value used in Lucene's implementation of BM25. */ + public static final float DEFAULT_K1 = 1.2f; + + /** Empirical value of c derived from modeling term frequency in English corpuses. */ + public static final float DEFAULT_C = 0.917f; + + /** Default k3 value. A negative value disables query-term frequency saturation. */ + public static final float DEFAULT_K3 = -1f; + + /** Multiplicative constant to term matching probability. Non-adjustable by users. */ + private static final float M = 0.00781f; + + /** + * Default term length if term BytesRef can't be decoded with UTF_8. This is close to the average + * word length in English. + */ + private static final float NON_UTF_8_DEFAULT_TERM_LENGTH = 5; + + /** Cache of decoded bytes. */ + private static final float[] LENGTH_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) { + LENGTH_TABLE[i] = SmallFloat.byte4ToInt((byte) i); + } + } + + /** Controls non-linear term frequency normalization (saturation). */ + private final float k1; + + /** Controls how much term length affects term rarity. */ + private final float c; + + /** + * Controls query-side term frequency saturation. A negative value disables saturation (linear + * behavior). + */ + private final float k3; + + /** + * StableTFL with the supplied parameter values. + * + * @param k1 Controls non-linear term frequency normalization (saturation). + * @param c Controls how much term length affects term rarity (decay constant). + * @param k3 Controls query-side term frequency saturation. When duplicate terms appear in a + * query, their boost is computed as ((k3 + 1) * qtf) / (k3 + qtf) instead of linear summing. + * A negative value disables saturation (linear behavior). Common values are 7 or 8. + * @throws IllegalArgumentException if {@code k1} is infinite or negative + * @throws IllegalArgumentException if {@code c} is infinite or negative + * @throws IllegalArgumentException if {@code k3} is NaN or infinite Review Comment: you are right. updated! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
