msokolov commented on code in PR #16383:
URL: https://github.com/apache/lucene/pull/16383#discussion_r3561410827
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java:
##########
@@ -211,6 +214,17 @@ private void writeFloat32Vectors(FlatFieldVectorsWriter<?>
fieldWriter, FieldInf
}
}
+ private void writeFloat16Vectors(FlatFieldVectorsWriter<?> fieldWriter,
FieldInfo fieldInfo)
+ throws IOException {
+ final ByteBuffer buffer =
+ ByteBuffer.allocate(fieldInfo.getVectorDimension() * Short.BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN);
+ for (Object v : fieldWriter.getVectors()) {
+ buffer.asShortBuffer().put((short[]) v);
+ vectorData.writeBytes(buffer.array(), buffer.array().length);
Review Comment:
do we not have a `IndexOutput.writeShorts`?
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
+ *
+ * @param name field name
+ * @param vector value
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(
+ String name, short[] vector, VectorSimilarityFunction
similarityFunction) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a new KnnFloatVectorField with the specified name, vector,
similarity function, and
+ * encoding.
+ *
+ * @param name the field name
+ * @param vector the float vector value
+ * @param similarityFunction the similarity function to use for vector
comparisons
+ * @param vectorEncoding the encoding format for the vector
+ */
+ public KnnFloat16VectorField(
+ String name,
+ short[] vector,
+ VectorSimilarityFunction similarityFunction,
+ VectorEncoding vectorEncoding) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2)
similarity. Fields are
+ * single-valued: each document has either one value or no value. Vectors of
a single field share
+ * the same dimension and similarity function.
+ *
+ * @param name field name
+ * @param vector value
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(String name, short[] vector) {
+ this(name, vector, VectorSimilarityFunction.EUCLIDEAN);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function.
+ *
+ * @param name field name
+ * @param vector value
+ * @param fieldType field type
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(String name, short[] vector, FieldType
fieldType) {
+ super(name, fieldType);
+ if (fieldType.vectorEncoding() != VectorEncoding.FLOAT16) {
+ throw new IllegalArgumentException(
+ "Attempt to create a vector for field "
+ + name
+ + " using float[] but the field encoding is "
Review Comment:
should read `short[]`
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java:
##########
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene95;
Review Comment:
it seems kind of funny to be adding to the `lucene95` codec, but I guess
that's where the other OffHeap*VectorValues continue to live?
##########
lucene/core/src/test/org/apache/lucene/search/TestVectorScorer.java:
##########
@@ -77,6 +83,12 @@ private Directory getIndexStore(String field, VectorEncoding
encoding, float[]..
v[j] = (byte) contents[i][j];
}
doc.add(new KnnByteVectorField(field, v, EUCLIDEAN));
+ } else if (encoding == VectorEncoding.FLOAT16) {
+ short[] v = new short[contents[i].length];
+ for (int j = 0; j < v.length; j++) {
+ v[j] = Float.floatToFloat16(contents[i][j]);
+ }
+ doc.add(new KnnFloat16VectorField(field, v, EUCLIDEAN));
Review Comment:
why EUCLIDEAN? But the case for `float` below doesn't specify any function?
##########
lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsReader.java:
##########
@@ -168,6 +169,11 @@ public ByteVectorValues getByteVectorValues(String field) {
throw new UnsupportedOperationException("Byte vectors not supported");
}
+ @Override
+ public Float16VectorValues getFloat16VectorValues(String field) throws
IOException {
+ throw new UnsupportedOperationException("Float16 vectors not supported");
Review Comment:
I wonder if it could support it? Not for this PR, but maybe it already has
such a thing?
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java:
##########
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene95;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+
+/** Read the vector values from the index input. This supports both iterated
and random access. */
+public abstract class OffHeapFloat16VectorValues extends Float16VectorValues
+ implements HasIndexSlice {
+
+ protected final int dimension;
+ protected final int size;
+ protected final IndexInput slice;
+ protected final int byteSize;
+ protected int lastOrd = -1;
+ protected final short[] value;
+ protected final VectorSimilarityFunction similarityFunction;
+ protected final FlatVectorsScorer flatVectorsScorer;
+
+ OffHeapFloat16VectorValues(
+ int dimension,
+ int size,
+ IndexInput slice,
+ int byteSize,
+ FlatVectorsScorer flatVectorsScorer,
+ VectorSimilarityFunction similarityFunction) {
+ this.dimension = dimension;
+ this.size = size;
+ this.slice = slice;
+ this.byteSize = byteSize;
+ this.similarityFunction = similarityFunction;
+ this.flatVectorsScorer = flatVectorsScorer;
+ value = new short[dimension];
+ }
+
+ @Override
+ public int dimension() {
+ return dimension;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public IndexInput getSlice() {
+ return slice;
+ }
+
+ @Override
+ public short[] vectorValue(int targetOrd) throws IOException {
+ if (lastOrd == targetOrd) {
+ return value;
+ }
+ slice.seek((long) targetOrd * byteSize);
+ slice.readShorts(value, 0, dimension);
+ lastOrd = targetOrd;
+ return value;
+ }
+
+ @Override
+ public VectorEncoding getEncoding() {
+ return VectorEncoding.FLOAT16;
+ }
+
+ public static OffHeapFloat16VectorValues load(
+ VectorSimilarityFunction vectorSimilarityFunction,
+ FlatVectorsScorer flatVectorsScorer,
+ OrdToDocDISIReaderConfiguration configuration,
+ VectorEncoding vectorEncoding,
+ int dimension,
+ long vectorDataOffset,
+ long vectorDataLength,
+ IndexInput vectorData)
+ throws IOException {
+ if (configuration.docsWithFieldOffset == -2 || vectorEncoding !=
VectorEncoding.FLOAT16) {
Review Comment:
seems weird that we respond to requests for other `vectorEncoding` by
returning an empty values rather than raising an error -- is this consistent
with the other encodings?
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java:
##########
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene95;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+
+/** Read the vector values from the index input. This supports both iterated
and random access. */
+public abstract class OffHeapFloat16VectorValues extends Float16VectorValues
+ implements HasIndexSlice {
+
+ protected final int dimension;
+ protected final int size;
+ protected final IndexInput slice;
+ protected final int byteSize;
+ protected int lastOrd = -1;
+ protected final short[] value;
+ protected final VectorSimilarityFunction similarityFunction;
+ protected final FlatVectorsScorer flatVectorsScorer;
+
+ OffHeapFloat16VectorValues(
+ int dimension,
+ int size,
+ IndexInput slice,
+ int byteSize,
+ FlatVectorsScorer flatVectorsScorer,
+ VectorSimilarityFunction similarityFunction) {
+ this.dimension = dimension;
+ this.size = size;
+ this.slice = slice;
+ this.byteSize = byteSize;
+ this.similarityFunction = similarityFunction;
+ this.flatVectorsScorer = flatVectorsScorer;
+ value = new short[dimension];
+ }
+
+ @Override
+ public int dimension() {
+ return dimension;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public IndexInput getSlice() {
+ return slice;
+ }
+
+ @Override
+ public short[] vectorValue(int targetOrd) throws IOException {
+ if (lastOrd == targetOrd) {
+ return value;
+ }
+ slice.seek((long) targetOrd * byteSize);
+ slice.readShorts(value, 0, dimension);
+ lastOrd = targetOrd;
+ return value;
+ }
+
+ @Override
+ public VectorEncoding getEncoding() {
+ return VectorEncoding.FLOAT16;
+ }
+
+ public static OffHeapFloat16VectorValues load(
+ VectorSimilarityFunction vectorSimilarityFunction,
+ FlatVectorsScorer flatVectorsScorer,
+ OrdToDocDISIReaderConfiguration configuration,
+ VectorEncoding vectorEncoding,
+ int dimension,
+ long vectorDataOffset,
+ long vectorDataLength,
+ IndexInput vectorData)
+ throws IOException {
+ if (configuration.docsWithFieldOffset == -2 || vectorEncoding !=
VectorEncoding.FLOAT16) {
+ return new EmptyOffHeapVectorValues(dimension, flatVectorsScorer,
vectorSimilarityFunction);
+ }
+ IndexInput bytesSlice = vectorData.slice("vector-data", vectorDataOffset,
vectorDataLength);
+ int byteSize = dimension * Short.BYTES;
+
+ if (configuration.docsWithFieldOffset == -1) {
+ return new DenseOffHeapVectorValues(
+ dimension,
+ configuration.size,
+ bytesSlice,
+ byteSize,
+ flatVectorsScorer,
+ vectorSimilarityFunction);
+ } else {
+ return new SparseOffHeapVectorValues(
+ configuration,
+ vectorData,
+ bytesSlice,
+ dimension,
+ byteSize,
+ flatVectorsScorer,
+ vectorSimilarityFunction);
+ }
+ }
+
+ /**
+ * Dense vector values that are stored off-heap. This is the most common
case when every doc has a
Review Comment:
maybe "This is the case" -- strike "most common" because it's the only one?
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java:
##########
@@ -339,6 +373,7 @@ private record FieldEntry(
switch (info.getVectorEncoding()) {
case BYTE -> Byte.BYTES;
case FLOAT32 -> Float.BYTES;
+ case FLOAT16 -> Short.BYTES;
Review Comment:
tiny nit, but it would make me happy to list these switch statements in
order of size: (BYTE, FLOAT16, FLOAT32); same throughout
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java:
##########
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene95;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+
+/** Read the vector values from the index input. This supports both iterated
and random access. */
+public abstract class OffHeapFloat16VectorValues extends Float16VectorValues
+ implements HasIndexSlice {
+
+ protected final int dimension;
+ protected final int size;
+ protected final IndexInput slice;
+ protected final int byteSize;
+ protected int lastOrd = -1;
+ protected final short[] value;
+ protected final VectorSimilarityFunction similarityFunction;
+ protected final FlatVectorsScorer flatVectorsScorer;
+
+ OffHeapFloat16VectorValues(
+ int dimension,
+ int size,
+ IndexInput slice,
+ int byteSize,
+ FlatVectorsScorer flatVectorsScorer,
+ VectorSimilarityFunction similarityFunction) {
+ this.dimension = dimension;
+ this.size = size;
+ this.slice = slice;
+ this.byteSize = byteSize;
+ this.similarityFunction = similarityFunction;
+ this.flatVectorsScorer = flatVectorsScorer;
+ value = new short[dimension];
+ }
+
+ @Override
+ public int dimension() {
+ return dimension;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public IndexInput getSlice() {
+ return slice;
+ }
+
+ @Override
+ public short[] vectorValue(int targetOrd) throws IOException {
+ if (lastOrd == targetOrd) {
+ return value;
+ }
+ slice.seek((long) targetOrd * byteSize);
+ slice.readShorts(value, 0, dimension);
+ lastOrd = targetOrd;
+ return value;
+ }
+
+ @Override
+ public VectorEncoding getEncoding() {
+ return VectorEncoding.FLOAT16;
+ }
+
+ public static OffHeapFloat16VectorValues load(
+ VectorSimilarityFunction vectorSimilarityFunction,
+ FlatVectorsScorer flatVectorsScorer,
+ OrdToDocDISIReaderConfiguration configuration,
+ VectorEncoding vectorEncoding,
+ int dimension,
+ long vectorDataOffset,
+ long vectorDataLength,
+ IndexInput vectorData)
+ throws IOException {
+ if (configuration.docsWithFieldOffset == -2 || vectorEncoding !=
VectorEncoding.FLOAT16) {
+ return new EmptyOffHeapVectorValues(dimension, flatVectorsScorer,
vectorSimilarityFunction);
+ }
+ IndexInput bytesSlice = vectorData.slice("vector-data", vectorDataOffset,
vectorDataLength);
+ int byteSize = dimension * Short.BYTES;
+
+ if (configuration.docsWithFieldOffset == -1) {
+ return new DenseOffHeapVectorValues(
+ dimension,
+ configuration.size,
+ bytesSlice,
+ byteSize,
+ flatVectorsScorer,
+ vectorSimilarityFunction);
+ } else {
+ return new SparseOffHeapVectorValues(
+ configuration,
+ vectorData,
+ bytesSlice,
+ dimension,
+ byteSize,
+ flatVectorsScorer,
+ vectorSimilarityFunction);
+ }
+ }
+
+ /**
+ * Dense vector values that are stored off-heap. This is the most common
case when every doc has a
+ * vector.
+ */
+ public static class DenseOffHeapVectorValues extends
OffHeapFloat16VectorValues {
+
+ public DenseOffHeapVectorValues(
+ int dimension,
+ int size,
+ IndexInput slice,
+ int byteSize,
+ FlatVectorsScorer flatVectorsScorer,
+ VectorSimilarityFunction similarityFunction) {
+ super(dimension, size, slice, byteSize, flatVectorsScorer,
similarityFunction);
+ }
+
+ @Override
+ public DenseOffHeapVectorValues copy() throws IOException {
+ return new DenseOffHeapVectorValues(
+ dimension, size, slice.clone(), byteSize, flatVectorsScorer,
similarityFunction);
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return ord;
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return acceptDocs;
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return createDenseIterator();
+ }
+
+ @Override
+ public VectorScorer scorer(short[] query) throws IOException {
+ DenseOffHeapVectorValues copy = copy();
+ DocIndexIterator iterator = copy.iterator();
+ RandomVectorScorer randomVectorScorer =
+ flatVectorsScorer.getRandomVectorScorer(similarityFunction, copy,
query);
+ return new VectorScorer() {
+ @Override
+ public float score() throws IOException {
+ return randomVectorScorer.score(iterator.docID());
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ return iterator;
+ }
+
+ @Override
+ public VectorScorer.Bulk bulk(DocIdSetIterator matchingDocs) {
+ return Bulk.fromRandomScorerDense(randomVectorScorer, iterator,
matchingDocs);
+ }
+ };
+ }
+ }
+
+ private static class SparseOffHeapVectorValues extends
OffHeapFloat16VectorValues {
+ private final DirectMonotonicReader ordToDoc;
+ private final IndexedDISI disi;
+ // dataIn was used to init a new IndexedDIS for #randomAccess()
Review Comment:
"IndexedDISI"
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java:
##########
@@ -251,6 +266,18 @@ private void writeSortedFloat32Vectors(
}
}
+ private void writeSortedFloat16Vectors(
+ FlatFieldVectorsWriter<?> fieldWriter, FieldInfo fieldInfo, int[]
ordMap) throws IOException {
+ final ByteBuffer buffer =
+ ByteBuffer.allocate(fieldInfo.getVectorDimension() * Short.BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN);
+ for (int ordinal : ordMap) {
+ short[] vector = (short[]) fieldWriter.getVectors().get(ordinal);
+ buffer.asShortBuffer().put(vector);
+ vectorData.writeBytes(buffer.array(), buffer.array().length);
Review Comment:
OK I checked and DataOutput's only bulk write method is for `byte[]`
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java:
##########
@@ -182,6 +183,7 @@ private static long alignOutput(IndexOutput output,
VectorEncoding encoding) thr
switch (encoding) {
case BYTE -> Float.BYTES;
case FLOAT32 -> 64; // optimal alignment for Arm Neoverse machines.
+ case FLOAT16 -> Float.BYTES;
Review Comment:
is this based on any testing? It seems like a good default, but just
wondering
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
Review Comment:
may be -> must, be or is
##########
lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java:
##########
@@ -68,6 +68,8 @@ public byte[] copyValue(byte[] vectorValue) {
}
};
break;
+ case FLOAT16:
+ throw new UnsupportedOperationException("FLOAT16 is not supported");
Review Comment:
Don't we need this class to support FLOAT16? I thought it was used during
indexing to hold vector values before `flush`?
##########
lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java:
##########
@@ -347,6 +379,26 @@ private static DocsWithFieldSet writeVectorData(
return docsWithField;
}
+ /**
+ * Writes the vector values to the output and returns a set of documents
that contains vectors.
+ */
+ private static DocsWithFieldSet writeFloat16VectorData(
+ IndexOutput output, Float16VectorValues float16VectorValues) throws
IOException {
+ DocsWithFieldSet docsWithField = new DocsWithFieldSet();
+ ByteBuffer buffer =
+ ByteBuffer.allocate(float16VectorValues.dimension() *
VectorEncoding.FLOAT16.byteSize)
+ .order(ByteOrder.LITTLE_ENDIAN);
+ KnnVectorValues.DocIndexIterator iter = float16VectorValues.iterator();
+ for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV =
iter.nextDoc()) {
+ // write vector
+ short[] value = float16VectorValues.vectorValue(iter.index());
+ buffer.asShortBuffer().put(value);
Review Comment:
should we allocate the ShortBuffer once, outside the loop? Not sure if it
matters
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
Review Comment:
`short[]`, I think?
##########
lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java:
##########
@@ -273,13 +312,19 @@ public static void mapOldOrdToNewOrd(
public static final class MergedVectorValues {
private MergedVectorValues() {}
- private static void validateFieldEncoding(FieldInfo fieldInfo,
VectorEncoding expected) {
+ private static void validateFieldEncoding(FieldInfo fieldInfo,
VectorEncoding... expected) {
assert fieldInfo != null && fieldInfo.hasVectorValues();
VectorEncoding fieldEncoding = fieldInfo.getVectorEncoding();
- if (fieldEncoding != expected) {
- throw new UnsupportedOperationException(
- "Cannot merge vectors encoded as [" + fieldEncoding + "] as " +
expected);
+ for (VectorEncoding exp : expected) {
Review Comment:
how come we now need to handle multiple VectorEncoding here?
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
+ *
+ * @param name field name
+ * @param vector value
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(
+ String name, short[] vector, VectorSimilarityFunction
similarityFunction) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a new KnnFloatVectorField with the specified name, vector,
similarity function, and
+ * encoding.
+ *
+ * @param name the field name
+ * @param vector the float vector value
+ * @param similarityFunction the similarity function to use for vector
comparisons
+ * @param vectorEncoding the encoding format for the vector
+ */
+ public KnnFloat16VectorField(
+ String name,
+ short[] vector,
+ VectorSimilarityFunction similarityFunction,
+ VectorEncoding vectorEncoding) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2)
similarity. Fields are
+ * single-valued: each document has either one value or no value. Vectors of
a single field share
+ * the same dimension and similarity function.
+ *
+ * @param name field name
+ * @param vector value
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(String name, short[] vector) {
+ this(name, vector, VectorSimilarityFunction.EUCLIDEAN);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function.
+ *
+ * @param name field name
+ * @param vector value
Review Comment:
"float16 vector value"
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
+ *
+ * @param name field name
+ * @param vector value
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(
+ String name, short[] vector, VectorSimilarityFunction
similarityFunction) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a new KnnFloatVectorField with the specified name, vector,
similarity function, and
+ * encoding.
+ *
+ * @param name the field name
+ * @param vector the float vector value
+ * @param similarityFunction the similarity function to use for vector
comparisons
+ * @param vectorEncoding the encoding format for the vector
+ */
+ public KnnFloat16VectorField(
+ String name,
+ short[] vector,
+ VectorSimilarityFunction similarityFunction,
+ VectorEncoding vectorEncoding) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2)
similarity. Fields are
+ * single-valued: each document has either one value or no value. Vectors of
a single field share
+ * the same dimension and similarity function.
+ *
+ * @param name field name
+ * @param vector value
Review Comment:
"float16 vector value"
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
+ *
+ * @param name field name
+ * @param vector value
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(
+ String name, short[] vector, VectorSimilarityFunction
similarityFunction) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a new KnnFloatVectorField with the specified name, vector,
similarity function, and
+ * encoding.
+ *
+ * @param name the field name
+ * @param vector the float vector value
Review Comment:
"the float16 vector value"
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
Review Comment:
"the float16 vector"?
##########
lucene/core/src/java/org/apache/lucene/index/LeafReader.java:
##########
@@ -315,6 +323,50 @@ public final TopDocs searchNearestVectors(
return collector.topDocs();
}
+ /**
+ * Return the k nearest neighbor documents as determined by comparison of
their vector values for
+ * this field, to the given vector, by the field's similarity function. The
score of each document
+ * is derived from the vector similarity in a way that ensures scores are
positive and that a
+ * larger score corresponds to a higher ranking.
+ *
+ * <p>The search is allowed to be approximate, meaning the results are not
guaranteed to be the
+ * true k closest neighbors. For large values of k (for example when k is
close to the total
+ * number of documents), the search may also retrieve fewer than k documents.
+ *
+ * <p>The returned {@link TopDocs} will contain a {@link ScoreDoc} for each
nearest neighbor,
+ * sorted in order of their similarity to the query vector (decreasing
scores). The {@link
+ * TotalHits} contains the number of documents visited during the search. If
the search stopped
+ * early because it hit {@code visitedLimit}, it is indicated through the
relation {@code
+ * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}.
+ *
+ * @param field the vector field to search
+ * @param target the short vector-valued query
Review Comment:
the fp16 vector valued query, as a short[]?
##########
lucene/core/src/java/org/apache/lucene/index/LeafReader.java:
##########
@@ -344,6 +396,35 @@ public abstract void searchNearestVectors(
String field, float[] target, KnnCollector knnCollector, AcceptDocs
acceptDocs)
throws IOException;
+ /**
+ * Return the k nearest neighbor documents as determined by comparison of
their vector values for
+ * this field, to the given vector, by the field's similarity function. The
score of each document
+ * is derived from the vector similarity in a way that ensures scores are
positive and that a
+ * larger score corresponds to a higher ranking.
+ *
+ * <p>The search is allowed to be approximate, meaning the results are not
guaranteed to be the
+ * true k closest neighbors. For large values of k (for example when k is
close to the total
+ * number of documents), the search may also retrieve fewer than k documents.
+ *
+ * <p>The returned {@link TopDocs} will contain a {@link ScoreDoc} for each
nearest neighbor, in
+ * order of their similarity to the query vector (decreasing scores). The
{@link TotalHits}
+ * contains the number of documents visited during the search. If the search
stopped early because
+ * it hit {@code visitedLimit}, it is indicated through the relation {@code
+ * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}.
+ *
+ * <p>The behavior is undefined if the given field doesn't have KNN vectors
enabled on its {@link
+ * FieldInfo}. The return value is never {@code null}.
+ *
+ * @param field the vector field to search
+ * @param target the short vector-valued query
Review Comment:
"float16"
##########
lucene/core/src/java/org/apache/lucene/index/Float16VectorValues.java:
##########
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.document.KnnFloat16VectorField;
+import org.apache.lucene.search.VectorScorer;
+
+/**
+ * This class provides access to per-document float16 vector values indexed as
{@link
+ * KnnFloat16VectorField}.
+ *
+ * @lucene.experimental
+ */
+public abstract class Float16VectorValues extends KnnVectorValues {
+
+ /** Sole constructor */
+ protected Float16VectorValues() {}
+
+ /**
+ * Return the vector value for the given vector ordinal which must be in [0,
size() - 1],
+ * otherwise IndexOutOfBoundsException is thrown. The returned array may be
shared across calls.
+ *
+ * @return the vector value
+ */
+ public abstract short[] vectorValue(int ord) throws IOException;
+
+ @Override
+ public abstract Float16VectorValues copy() throws IOException;
+
+ /**
+ * Checks the Vector Encoding of a field
+ *
+ * @throws IllegalStateException if {@code field} has vectors, but using a
different encoding
+ * @lucene.internal
+ * @lucene.experimental
+ */
+ public static void checkField(LeafReader in, String field) {
+ FieldInfo fi = in.getFieldInfos().fieldInfo(field);
+ if (fi != null && fi.hasVectorValues() && fi.getVectorEncoding() !=
VectorEncoding.FLOAT16) {
+ throw new IllegalStateException(
+ "Unexpected vector encoding ("
+ + fi.getVectorEncoding()
+ + ") for field "
+ + field
+ + "(expected="
+ + VectorEncoding.FLOAT16
+ + ")");
+ }
+ }
+
+ /**
+ * Return a {@link VectorScorer} for the given query vector and the current
{@link
+ * Float16VectorValues}. When the underlying format quantizes the vectors,
this will return a
+ * {@link VectorScorer} that scores against the quantized vectors.
+ *
+ * @param target the query vector
+ * @return a {@link VectorScorer} instance or null
+ */
+ public VectorScorer scorer(short[] target) throws IOException {
+ throw new UnsupportedOperationException();
Review Comment:
hmm is this not needed to support search?
##########
lucene/core/src/java/org/apache/lucene/index/VectorEncoding.java:
##########
@@ -29,7 +29,10 @@ public enum VectorEncoding {
BYTE(1),
/** Encodes vector using 32 bits of precision per sample in IEEE floating
point format. */
- FLOAT32(4);
+ FLOAT32(4),
+
+ /** Encodes vector using 16 bits of precision per sample in IEEE floating
point format. */
Review Comment:
maybe "IEEE half-precision floating-point format"?
##########
lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java:
##########
@@ -68,6 +68,8 @@ public byte[] copyValue(byte[] vectorValue) {
}
};
break;
+ case FLOAT16:
+ throw new UnsupportedOperationException("FLOAT16 is not supported");
Review Comment:
oh I see, this is only used for older codecs. What about SimpleText though?
Won't that use it also?
##########
lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java:
##########
Review Comment:
does randomVectorEncoding now sometimes produce FLOAT16?
##########
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstKnnFloat16ValueSource.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.queries.function.valuesource;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+
+/** Function that returns a constant float16 vector value for every document.
*/
+public class ConstKnnFloat16ValueSource extends ValueSource {
+ private final short[] vector;
+
+ public ConstKnnFloat16ValueSource(short[] constVector) {
+ this.vector = Objects.requireNonNull(constVector, "constVector");
+ }
+
+ @Override
+ public FunctionValues getValues(Map<Object, Object> context,
LeafReaderContext readerContext)
Review Comment:
do we really need these float16 values sources? I wonder if we could use
Float valuesSources over float16 vector fields?
##########
lucene/core/src/java/org/apache/lucene/document/KnnFloat16VectorField.java:
##########
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.document;
+
+import java.util.Objects;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.KnnFloat16VectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * A field that contains a single float16 numeric vector (or none) for each
document. Vectors are
+ * dense - that is, every dimension of a vector contains an explicit value,
stored packed into an
+ * array (of type short[]) whose length is the vector dimension. Values can be
retrieved using
+ * {@link Float16VectorValues}, which is a forward-only docID-based iterator
and also offers
+ * random-access by dense ordinal (not docId). {@link
VectorSimilarityFunction} may be used to
+ * compare vectors at query time (for example as part of result ranking). A
{@link
+ * KnnFloat16VectorField} may be associated with a search similarity function
defining the metric
+ * used for nearest-neighbor search among vectors of that field.
+ *
+ * @lucene.experimental
+ */
+public class KnnFloat16VectorField extends Field {
+
+ private static FieldType createType(short[] v, VectorSimilarityFunction
similarityFunction) {
+ if (v == null) {
+ throw new IllegalArgumentException("vector value must not be null");
+ }
+ int dimension = v.length;
+ if (dimension == 0) {
+ throw new IllegalArgumentException("cannot index an empty vector");
+ }
+ if (similarityFunction == null) {
+ throw new IllegalArgumentException("similarity function must not be
null");
+ }
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * A convenience method for creating a vector field type.
+ *
+ * @param dimension dimension of vectors
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or has
dimension > 1024.
+ */
+ public static FieldType createFieldType(
+ int dimension, VectorSimilarityFunction similarityFunction) {
+ FieldType type = new FieldType();
+ type.setVectorAttributes(dimension, VectorEncoding.FLOAT16,
similarityFunction);
+ type.freeze();
+ return type;
+ }
+
+ /**
+ * Create a new vector query for the provided field targeting the float
vector
+ *
+ * @param field The field to query
+ * @param queryVector The float vector target
+ * @param k The number of nearest neighbors to gather
+ * @return A new vector query
+ */
+ public static Query newVectorQuery(String field, short[] queryVector, int k)
{
+ return new KnnFloat16VectorQuery(field, queryVector, k);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function. Note that
+ * some vector similarities (like {@link
VectorSimilarityFunction#DOT_PRODUCT}) require values to
+ * be unit-length, which can be enforced using {@link
VectorUtil#l2normalize(float[])}.
+ *
+ * @param name field name
+ * @param vector value
+ * @param similarityFunction a function defining vector proximity.
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(
+ String name, short[] vector, VectorSimilarityFunction
similarityFunction) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a new KnnFloatVectorField with the specified name, vector,
similarity function, and
+ * encoding.
+ *
+ * @param name the field name
+ * @param vector the float vector value
+ * @param similarityFunction the similarity function to use for vector
comparisons
+ * @param vectorEncoding the encoding format for the vector
+ */
+ public KnnFloat16VectorField(
+ String name,
+ short[] vector,
+ VectorSimilarityFunction similarityFunction,
+ VectorEncoding vectorEncoding) {
+ super(name, createType(vector, similarityFunction));
+ fieldsData = vector; // null check done above
+ }
+
+ /**
+ * Creates a numeric vector field with the default EUCLIDEAN_HNSW (L2)
similarity. Fields are
+ * single-valued: each document has either one value or no value. Vectors of
a single field share
+ * the same dimension and similarity function.
+ *
+ * @param name field name
+ * @param vector value
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
+ * dimension > 1024.
+ */
+ public KnnFloat16VectorField(String name, short[] vector) {
+ this(name, vector, VectorSimilarityFunction.EUCLIDEAN);
+ }
+
+ /**
+ * Creates a numeric vector field. Fields are single-valued: each document
has either one value or
+ * no value. Vectors of a single field share the same dimension and
similarity function.
+ *
+ * @param name field name
+ * @param vector value
+ * @param fieldType field type
+ * @throws IllegalArgumentException if any parameter is null, or the vector
is empty or has
Review Comment:
I don't think this throws when the dimension > 1024; let's fix all the other
comments as well? This used to be true, but enforcement got moved down into the
codec and these methods no longer do it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
