danmuzi commented on code in PR #16383: URL: https://github.com/apache/lucene/pull/16383#discussion_r3581713418
########## lucene/core/src/java/org/apache/lucene/search/KnnFloat16VectorQuery.java: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import static org.apache.lucene.search.knn.KnnSearchStrategy.Hnsw.DEFAULT; + +import java.io.IOException; +import java.util.Arrays; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.document.KnnFloat16VectorField; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.knn.KnnCollectorManager; +import org.apache.lucene.search.knn.KnnSearchStrategy; +import org.apache.lucene.util.ArrayUtil; + +/** + * Uses {@link KnnVectorsReader#search(String, short[], KnnCollector, AcceptDocs)} to perform + * nearest neighbour search. + * + * <p>This query also allows for performing a kNN search subject to a filter. In this case, it first + * executes the filter for each leaf, then chooses a strategy dynamically: + * + * <ul> + * <li>If the filter cost is less than k, just execute an exact search + * <li>Otherwise run a kNN search subject to the filter + * <li>If the kNN search visits too many vectors without completing, stop and run an exact search + * </ul> + */ +public class KnnFloat16VectorQuery extends AbstractKnnVectorQuery { + + private static final TopDocs NO_RESULTS = TopDocsCollector.EMPTY_TOPDOCS; + + protected final short[] target; + + /** + * Find the <code>k</code> nearest documents to the target vector according to the vectors in the + * given field. <code>target</code> vector. + * + * @param field a field that has been indexed as a {@link KnnFloat16VectorField}. + * @param target the target of the search + * @param k the number of documents to find + * @throws IllegalArgumentException if <code>k</code> is less than 1 + */ + public KnnFloat16VectorQuery(String field, short[] target, int k) { + this(field, target, k, null); + } + + /** + * Find the <code>k</code> nearest documents to the target vector according to the vectors in the + * given field. <code>target</code> vector. + * + * @param field a field that has been indexed as a {@link KnnFloat16VectorField}. + * @param target the target of the search + * @param k the number of documents to find + * @param filter a filter applied before the vector search + * @throws IllegalArgumentException if <code>k</code> is less than 1 + */ + public KnnFloat16VectorQuery(String field, short[] target, int k, Query filter) { + this(field, target, k, filter, DEFAULT); + } + + /** + * Find the <code>k</code> nearest documents to the target vector according to the vectors in the + * given field. <code>target</code> vector. + * + * @param field a field that has been indexed as a {@link KnnFloat16VectorField}. + * @param target the target of the search + * @param k the number of documents to find + * @param filter a filter applied before the vector search + * @param searchStrategy the search strategy to use. If null, the default strategy will be used. + * The underlying format may not support all strategies and is free to ignore the requested + * strategy. + * @lucene.experimental + */ + public KnnFloat16VectorQuery( + String field, short[] target, int k, Query filter, KnnSearchStrategy searchStrategy) { + super(field, k, filter, searchStrategy); + this.target = target; Review Comment: The float32 implementation validates the target vector as follows: ```java this.target = VectorUtil.checkFinite(Objects.requireNonNull(target, "target")); ``` This class does not have any validation for `target`, so it can cause several issues. 1) If `target` is null, NPE is thrown later during search from `DefaultFlatVectorScorer.getRandomVectorScorer()` when accessing `target.length`. 2) If the `target` contains values such as 0x7C00(+Infinity), 0xFC00(-Infinity), or 0x7E00 (NaN), `similarityFunction.compare()` may return `NaN`. In `TopKnnCollector`, comparisons involving NaN return false, so the search may silently return incorrect top-K results without throwing an exception. This is only caught when assertions are enabled, by the assert `Float.isFinite(result)` check around `VectorUtil.dotProduct()`. In production, the issue may remain undetected in production level. 3) If `target.length` is 0, `KnnFloat16VectorQuery.toString()` throws an `ArrayIndexOutOfBoundsException` because it accesses `target[0]`. So I think adding `VectorUtil.checkFiniteFloat16()` would be a reasonable solution. ```java // VectorUtil.java public static short[] checkFiniteFloat16(short[] v) { for (int i = 0; i < v.length; i++) { if ((v[i] & 0x7C00) == 0x7C00) { throw new IllegalArgumentException( "non-finite float16 value at vector[" + i + "]=" + Float.float16ToFloat(v[i])); } } return v; } // KnnFloat16VectorQuery.java this.target = VectorUtil.checkFiniteFloat16(Objects.requireNonNull(target, "target")); ``` ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. + * + * @param name field name + * @param vector value + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField( + String name, short[] vector, VectorSimilarityFunction similarityFunction) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a new KnnFloatVectorField with the specified name, vector, similarity function, and + * encoding. + * + * @param name the field name + * @param vector the float vector value + * @param similarityFunction the similarity function to use for vector comparisons + * @param vectorEncoding the encoding format for the vector + */ + public KnnFloat16VectorField( + String name, + short[] vector, + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding) { Review Comment: It seems not used. ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. + * + * @param name field name + * @param vector value + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField( + String name, short[] vector, VectorSimilarityFunction similarityFunction) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a new KnnFloatVectorField with the specified name, vector, similarity function, and + * encoding. + * + * @param name the field name + * @param vector the float vector value + * @param similarityFunction the similarity function to use for vector comparisons + * @param vectorEncoding the encoding format for the vector + */ + public KnnFloat16VectorField( + String name, + short[] vector, + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2) similarity. Fields are + * single-valued: each document has either one value or no value. Vectors of a single field share + * the same dimension and similarity function. + * + * @param name field name + * @param vector value + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField(String name, short[] vector) { + this(name, vector, VectorSimilarityFunction.EUCLIDEAN); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. + * + * @param name field name + * @param vector value + * @param fieldType field type + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField(String name, short[] vector, FieldType fieldType) { + super(name, fieldType); + if (fieldType.vectorEncoding() != VectorEncoding.FLOAT16) { + throw new IllegalArgumentException( + "Attempt to create a vector for field " + + name + + " using float[] but the field encoding is " + + fieldType.vectorEncoding()); + } + Objects.requireNonNull(vector, "vector value must not be null"); + if (vector.length != fieldType.vectorDimension()) { + throw new IllegalArgumentException( + "The number of vector dimensions does not match the field type"); + } + fieldsData = vector; Review Comment: Ditto. ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. Review Comment: Following up on Michael's comment, `VectorUtil` does not have a `l2normalize` method for `short[]`. ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. + * + * @param name field name + * @param vector value + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField( + String name, short[] vector, VectorSimilarityFunction similarityFunction) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above Review Comment: This should also validate the vector using the `VectorUtil.checkFiniteFloat16()` suggested in the comment on `KnnFloat16VectorQuery`. ```java fieldsData = VectorUtil.checkFiniteFloat16(vector); ``` Without this validation, the following issues may occur: 1) When the segment is flushed, the HNSW graph builder computes similarity scores with these values, and the results become `NaN`. `NaN` breaks the neighbor selection logic, so the graph connections around those nodes end up more or less random. 2) Once these values are written into a segment, every later merge reuses them to rebuild the graph, so the damage spreads to new segments as well. 3) No error or warning is raised in production. ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. + * + * @param name field name + * @param vector value + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField( + String name, short[] vector, VectorSimilarityFunction similarityFunction) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a new KnnFloatVectorField with the specified name, vector, similarity function, and + * encoding. + * + * @param name the field name + * @param vector the float vector value + * @param similarityFunction the similarity function to use for vector comparisons + * @param vectorEncoding the encoding format for the vector + */ + public KnnFloat16VectorField( + String name, + short[] vector, + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above Review Comment: Ditto. ########## lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java: ########## @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.document; + +import java.util.Objects; +import org.apache.lucene.index.Float16VectorValues; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnFloat16VectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.VectorUtil; + +/** + * A field that contains a single float16 numeric vector (or none) for each document. Vectors are + * dense - that is, every dimension of a vector contains an explicit value, stored packed into an + * array (of type short[]) whose length is the vector dimension. Values can be retrieved using + * {@link Float16VectorValues}, which is a forward-only docID-based iterator and also offers + * random-access by dense ordinal (not docId). {@link VectorSimilarityFunction} may be used to + * compare vectors at query time (for example as part of result ranking). A {@link + * KnnFloat16VectorField} may be associated with a search similarity function defining the metric + * used for nearest-neighbor search among vectors of that field. + * + * @lucene.experimental + */ +public class KnnFloat16VectorField extends Field { + + private static FieldType createType(short[] v, VectorSimilarityFunction similarityFunction) { + if (v == null) { + throw new IllegalArgumentException("vector value must not be null"); + } + int dimension = v.length; + if (dimension == 0) { + throw new IllegalArgumentException("cannot index an empty vector"); + } + if (similarityFunction == null) { + throw new IllegalArgumentException("similarity function must not be null"); + } + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * A convenience method for creating a vector field type. + * + * @param dimension dimension of vectors + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or has dimension > 1024. + */ + public static FieldType createFieldType( + int dimension, VectorSimilarityFunction similarityFunction) { + FieldType type = new FieldType(); + type.setVectorAttributes(dimension, VectorEncoding.FLOAT16, similarityFunction); + type.freeze(); + return type; + } + + /** + * Create a new vector query for the provided field targeting the float vector + * + * @param field The field to query + * @param queryVector The float vector target + * @param k The number of nearest neighbors to gather + * @return A new vector query + */ + public static Query newVectorQuery(String field, short[] queryVector, int k) { + return new KnnFloat16VectorQuery(field, queryVector, k); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. Note that + * some vector similarities (like {@link VectorSimilarityFunction#DOT_PRODUCT}) require values to + * be unit-length, which can be enforced using {@link VectorUtil#l2normalize(float[])}. + * + * @param name field name + * @param vector value + * @param similarityFunction a function defining vector proximity. + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField( + String name, short[] vector, VectorSimilarityFunction similarityFunction) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a new KnnFloatVectorField with the specified name, vector, similarity function, and + * encoding. + * + * @param name the field name + * @param vector the float vector value + * @param similarityFunction the similarity function to use for vector comparisons + * @param vectorEncoding the encoding format for the vector + */ + public KnnFloat16VectorField( + String name, + short[] vector, + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding) { + super(name, createType(vector, similarityFunction)); + fieldsData = vector; // null check done above + } + + /** + * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2) similarity. Fields are + * single-valued: each document has either one value or no value. Vectors of a single field share + * the same dimension and similarity function. + * + * @param name field name + * @param vector value + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField(String name, short[] vector) { + this(name, vector, VectorSimilarityFunction.EUCLIDEAN); + } + + /** + * Creates a numeric vector field. Fields are single-valued: each document has either one value or + * no value. Vectors of a single field share the same dimension and similarity function. + * + * @param name field name + * @param vector value + * @param fieldType field type + * @throws IllegalArgumentException if any parameter is null, or the vector is empty or has + * dimension > 1024. + */ + public KnnFloat16VectorField(String name, short[] vector, FieldType fieldType) { + super(name, fieldType); + if (fieldType.vectorEncoding() != VectorEncoding.FLOAT16) { + throw new IllegalArgumentException( + "Attempt to create a vector for field " + + name + + " using float[] but the field encoding is " + + fieldType.vectorEncoding()); + } + Objects.requireNonNull(vector, "vector value must not be null"); + if (vector.length != fieldType.vectorDimension()) { + throw new IllegalArgumentException( + "The number of vector dimensions does not match the field type"); + } + fieldsData = vector; + } + + /** Return the vector value of this field */ + public short[] vectorValue() { + return (short[]) fieldsData; + } + + /** + * Set the vector value of this field + * + * @param value the value to set; must not be null, and length must match the field type + */ + public void setVectorValue(short[] value) { + if (value == null) { + throw new IllegalArgumentException("value must not be null"); + } + if (value.length != type.vectorDimension()) { + throw new IllegalArgumentException( + "value length " + value.length + " must match field dimension " + type.vectorDimension()); + } + fieldsData = value; Review Comment: Ditto. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
