rahil-c commented on code in PR #19310: URL: https://github.com/apache/hudi/pull/19310#discussion_r3839883130
########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorIndexOptions.java: ########## @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Options accepted by {@code CREATE INDEX ... USING VECTOR}. + * + * <p>The indexed column's Hudi {@code VECTOR(D[, elementType])} schema is authoritative for + * dimension and element type. Index options configure only the acceleration structure. DDL + * implementations must call {@link #validateAndNormalize(Map)} before persisting an index + * definition; individual parsing helpers are intentionally private so aggregate validation cannot + * be bypassed. + */ +public final class VectorIndexOptions { + + public static final String METRIC = "vector.metric"; + public static final String QUANTIZER = "vector.quantizer"; + public static final String NUM_CLUSTERS = "vector.num_clusters"; + public static final String MAX_ITER = "vector.max_iter"; + public static final String RABITQ_BITS = "vector.rabitq.bits"; + public static final String RABITQ_SEED = "vector.rabitq.seed"; + public static final String RABITQ_ASSUME_NORMALIZED = "vector.rabitq.assume_normalized"; + public static final String QUERY_NUM_PROBES = "vector.query.nprobes"; + public static final String QUERY_REFINE_FACTOR = "vector.query.refine_factor"; + public static final String QUERY_MODE = "vector.query.mode"; + public static final String QUERY_STALE_POLICY = "vector.query.stale_policy"; + + public static final VectorDistanceMetric DEFAULT_METRIC = VectorDistanceMetric.COSINE; + public static final VectorQuantizer DEFAULT_QUANTIZER = VectorQuantizer.IVF_RABITQ; + public static final int DEFAULT_NUM_CLUSTERS = 256; + public static final int DEFAULT_MAX_ITER = 20; + public static final int DEFAULT_RABITQ_BITS = 4; + public static final long DEFAULT_RABITQ_SEED = 42L; + public static final int DEFAULT_NUM_PROBES = 32; + public static final int DEFAULT_REFINE_FACTOR = 50; + public static final VectorQueryMode DEFAULT_QUERY_MODE = VectorQueryMode.EXACT_RERANK; + public static final VectorStalePolicy DEFAULT_STALE_POLICY = VectorStalePolicy.FAIL; + + private static final Set<String> SUPPORTED_OPTIONS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + METRIC, + QUANTIZER, + NUM_CLUSTERS, + MAX_ITER, + RABITQ_BITS, + RABITQ_SEED, + RABITQ_ASSUME_NORMALIZED, + QUERY_NUM_PROBES, + QUERY_REFINE_FACTOR, + QUERY_MODE, + QUERY_STALE_POLICY))); + + private VectorIndexOptions() { + } + + /** + * Validates the complete option map and returns canonical values for persistence. + * + * <p>The returned map contains every supported option, including explicit defaults. Unknown, + * retired, misspelled, and invalid options are rejected instead of being silently ignored. + */ + public static Map<String, String> validateAndNormalize(Map<String, String> options) { + Set<String> unknownOptions = new HashSet<>(options.keySet()); + unknownOptions.removeAll(SUPPORTED_OPTIONS); + if (!unknownOptions.isEmpty()) { + throw new IllegalArgumentException("Unsupported vector index options: " + unknownOptions); + } + + VectorDistanceMetric metric = getMetric(options); + VectorQuantizer quantizer = getQuantizer(options); + int numClusters = getNumClusters(options); + int maxIter = getMaxIter(options); + int bits = getRaBitQBits(options); + long seed = getRaBitQSeed(options); + boolean assumeNormalized = shouldAssumeNormalizedVectors(options); + int numProbes = getNumProbes(options); + int refineFactor = getRefineFactor(options); + VectorQueryMode queryMode = getQueryMode(options); + VectorStalePolicy stalePolicy = getStalePolicy(options); + + if (numProbes > numClusters) { Review Comment: `nprobes` defaults to 32, so any `num_clusters` below 32 fails here even if the user never set `nprobes`. Clamp the default to `min(DEFAULT_NUM_PROBES, numClusters)`? ########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorDistanceMetric.java: ########## @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Locale; + +/** + * Distance metrics for vector similarity search. + * + * <p>All metrics are returned as distances (smaller = more similar), + * so they can be compared uniformly with a min-heap. + */ +public enum VectorDistanceMetric { Review Comment: No test for this class here, `TestVectorDistanceMetric` is in #19318. If this merges alone the maths is uncovered. ########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorQueryMode.java: ########## @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Locale; + +/** Supported vector-query execution modes. */ +public enum VectorQueryMode { + APPROXIMATE, + EXACT_RERANK; + + static VectorQueryMode fromString(String value) { Review Comment: These four normalize differently, so `'dot product'` works but `'exact rerank'` doesn't. Share one helper? ########## hudi-common/src/test/java/org/apache/hudi/common/index/vector/TestVectorIndexOptions.java: ########## @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TestVectorIndexOptions { + + @Test + void testDefaultsAreCanonicalAndComplete() { + assertEquals( + opts( + VectorIndexOptions.METRIC, "cosine", + VectorIndexOptions.QUANTIZER, "IVF_RABITQ", + VectorIndexOptions.NUM_CLUSTERS, "256", + VectorIndexOptions.MAX_ITER, "20", + VectorIndexOptions.RABITQ_BITS, "4", + VectorIndexOptions.RABITQ_SEED, "42", + VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "false", + VectorIndexOptions.QUERY_NUM_PROBES, "32", + VectorIndexOptions.QUERY_REFINE_FACTOR, "50", + VectorIndexOptions.QUERY_MODE, "exact_rerank", + VectorIndexOptions.QUERY_STALE_POLICY, "fail"), + VectorIndexOptions.validateAndNormalize(opts())); + } + + @Test + void testValuesAreNormalizedForPersistence() { + Map<String, String> normalized = VectorIndexOptions.validateAndNormalize(opts( + VectorIndexOptions.METRIC, "DOT-PRODUCT", + VectorIndexOptions.QUANTIZER, "ivf-rabitq", + VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "TRUE", + VectorIndexOptions.QUERY_MODE, "EXACT-RERANK", + VectorIndexOptions.QUERY_STALE_POLICY, "WARN")); + + assertEquals("dot_product", normalized.get(VectorIndexOptions.METRIC)); + assertEquals("IVF_RABITQ", normalized.get(VectorIndexOptions.QUANTIZER)); + assertEquals("true", normalized.get(VectorIndexOptions.RABITQ_ASSUME_NORMALIZED)); + assertEquals("exact_rerank", normalized.get(VectorIndexOptions.QUERY_MODE)); + assertEquals("warn", normalized.get(VectorIndexOptions.QUERY_STALE_POLICY)); + assertThrows( + UnsupportedOperationException.class, + () -> normalized.put(VectorIndexOptions.METRIC, "l2")); + } + + @Test + void testEveryMetricQueryModeAndStalePolicyIsAccepted() { + assertCanonical(VectorIndexOptions.METRIC, "cosine", "cosine"); + assertCanonical(VectorIndexOptions.METRIC, "l2", "l2"); + assertCanonical(VectorIndexOptions.METRIC, "dot_product", "dot_product"); + assertCanonical(VectorIndexOptions.QUERY_MODE, "approximate", "approximate"); + assertCanonical(VectorIndexOptions.QUERY_MODE, "exact_rerank", "exact_rerank"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "fail", "fail"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "warn", "warn"); + assertCanonical(VectorIndexOptions.QUERY_STALE_POLICY, "fallback", "fallback"); + } + + @Test + void testUnknownRetiredAndMisspelledOptionsAreRejected() { + assertInvalidOption("vector.dimension", "128"); + assertInvalidOption("vector.query.nprobe", "8"); + assertInvalidOption("vector.unknown", "value"); + } + + @Test + void testUnsupportedEnumValuesAreRejectedWithOptionContext() { + assertInvalidValueContainsKey(VectorIndexOptions.METRIC, "manhattan"); + assertInvalidValueContainsKey(VectorIndexOptions.QUANTIZER, "pq"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_MODE, "fast-ish"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_STALE_POLICY, "ignore"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_ASSUME_NORMALIZED, "yes"); + } + + @Test + void testNumericOptionsAreValidatedWithOptionContext() { + assertCanonical(VectorIndexOptions.RABITQ_BITS, "1", "1"); + assertCanonical(VectorIndexOptions.RABITQ_BITS, "8", "8"); + assertInvalidValueContainsKey(VectorIndexOptions.NUM_CLUSTERS, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.MAX_ITER, "-1"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_NUM_PROBES, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.QUERY_REFINE_FACTOR, "-1"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_BITS, "0"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_BITS, "9"); + assertInvalidValueContainsKey(VectorIndexOptions.RABITQ_SEED, "many"); + } + + @Test + void testNumProbesMustNotExceedNumClusters() { Review Comment: Both cases set both options, so the default path isn't covered. Setting only `num_clusters` would have caught it. ########## hudi-common/src/main/java/org/apache/hudi/common/index/vector/VectorDistanceMetric.java: ########## @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.common.index.vector; + +import java.util.Locale; + +/** + * Distance metrics for vector similarity search. + * + * <p>All metrics are returned as distances (smaller = more similar), + * so they can be compared uniformly with a min-heap. + */ +public enum VectorDistanceMetric { + + /** + * Cosine distance: 1 - cosine_similarity. + * Range: [0, 2]. 0 = identical direction, 2 = opposite. + */ + COSINE { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double dot = 0; + double normA = 0; + double normB = 0; + for (int i = 0; i < a.length; i++) { + dot += (double) a[i] * b[i]; + normA += (double) a[i] * a[i]; + normB += (double) b[i] * b[i]; + } + double denom = Math.sqrt(normA) * Math.sqrt(normB); + return denom == 0.0 ? 1.0f : (float) (1.0 - dot / denom); + } + }, + + /** + * Euclidean (L2) distance. + * Range: [0, ∞). 0 = identical. + */ + L2 { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double sum = 0; + for (int i = 0; i < a.length; i++) { + double d = (double) a[i] - b[i]; + sum += d * d; + } + return (float) Math.sqrt(sum); + } + }, + + /** + * Maximum inner product distance: negated dot product. + * Negated so smaller = higher similarity, consistent with the min-heap contract. + */ + DOT_PRODUCT { + @Override + public float compute(float[] a, float[] b) { + checkDimensions(a, b); + double dot = 0; + for (int i = 0; i < a.length; i++) { + dot += (double) a[i] * b[i]; + } + return (float) -dot; + } + }; + + /** + * Compute the distance between two float vectors. + * + * @param a first vector + * @param b second vector + * @return non-negative distance (smaller = more similar) + */ + public abstract float compute(float[] a, float[] b); Review Comment: NaN propagates through all three metrics, and NaN comparisons are always false, so the min-heap ordering in the javadoc doesn't hold. Guard, or document finite-only inputs? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
