This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7998 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 09692b34afa8c9935a284b00d2e07456252bb091 Author: Andrew Purtell <[email protected]> AuthorDate: Thu Sep 17 12:32:35 2026 -0700 PHOENIX-7998 Phase 2: Distance functions and operators Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --- phoenix-client-parent/pom.xml | 38 ++ phoenix-core-client/pom.xml | 49 ++ phoenix-core-client/src/main/antlr3/PhoenixSQL.g | 26 +- .../apache/phoenix/exception/SQLExceptionCode.java | 1 + .../apache/phoenix/expression/ExpressionType.java | 6 +- .../function/CosineDistanceFunction.java | 73 +++ .../expression/function/DistanceFunction.java | 182 +++++++ .../expression/function/DistanceKernel.java | 46 ++ .../function/InnerProductDistanceFunction.java | 67 +++ .../expression/function/L2DistanceFunction.java | 64 +++ .../function/L2DistanceSquaredFunction.java | 75 +++ .../expression/function/ScalarDistanceKernel.java | 121 +++++ .../expression/function/VectorDistanceUtil.java | 77 +++ .../phoenix/parse/DistanceFunctionParseNode.java | 44 ++ .../org/apache/phoenix/parse/ParseNodeFactory.java | 15 + .../expression/function/DistanceKernel.java | 88 ++++ .../expression/function/PanamaDistanceKernel.java | 122 +++++ phoenix-core/pom.xml | 35 ++ .../expression/function/DistanceFunctionsTest.java | 531 ++++++++++++++++++++ .../function/VectorDistanceUtilTest.java | 264 ++++++++++ .../phoenix/parse/DistanceOperatorParseTest.java | 539 +++++++++++++++++++++ .../function/PanamaDistanceKernelParityTest.java | 109 +++++ phoenix-mapreduce-byo-shaded-hbase/pom.xml | 38 ++ phoenix-server/pom.xml | 38 ++ pom.xml | 8 + 25 files changed, 2654 insertions(+), 2 deletions(-) diff --git a/phoenix-client-parent/pom.xml b/phoenix-client-parent/pom.xml index 47b62405c3..d9b59422b6 100644 --- a/phoenix-client-parent/pom.xml +++ b/phoenix-client-parent/pom.xml @@ -267,4 +267,42 @@ </plugins> </pluginManagement> </build> + + <profiles> + <profile> + <id>java21</id> + <!-- Mark the shaded jar as multi-release. A Java 21 JVM then loads the SIMD distance kernels + of phoenix-core-client from META-INF/versions/21. --> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <filters combine.children="append"> + <!-- Keep only the versioned classes of Phoenix. Shade does not relocate the + versioned classes of bundled libraries. A multi-release jar serves those classes + under their original names, which do not match the relocated base classes. --> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>%regex[META-INF/versions/[0-9]+/(?!org/apache/phoenix/).*]</exclude> + </excludes> + </filter> + </filters> + <transformers combine.children="append"> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <manifestEntries> + <Multi-Release>true</Multi-Release> + </manifestEntries> + </transformer> + </transformers> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> + </profiles> </project> diff --git a/phoenix-core-client/pom.xml b/phoenix-core-client/pom.xml index 095b5dd5c7..d3211ea388 100644 --- a/phoenix-core-client/pom.xml +++ b/phoenix-core-client/pom.xml @@ -447,4 +447,53 @@ </plugin> </plugins> </build> + + <profiles> + <profile> + <id>java21</id> + <!-- Compile the SIMD distance kernels (Panama Vector API) into a multi-release jar. --> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>compile-java21</id> + <goals> + <goal>compile</goal> + </goals> + <phase>compile</phase> + <configuration> + <release>21</release> + <compileSourceRoots> + <compileSourceRoot>${project.basedir}/src/main/java21</compileSourceRoot> + </compileSourceRoots> + <multiReleaseOutput>true</multiReleaseOutput> + <compilerArgs> + <arg>--add-modules</arg> + <arg>jdk.incubator.vector</arg> + </compilerArgs> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestEntries> + <Multi-Release>true</Multi-Release> + </manifestEntries> + </archive> + <excludes> + <exclude>META-INF/versions/*/META-INF/**</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g index c2c49ace51..d7e8db78b6 100644 --- a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g @@ -1196,7 +1196,20 @@ bind_expression returns [BindParseNode ret] ; value_expression returns [ParseNode ret] - : i=add_expression { $ret = i; } + : i=distance_expression { $ret = i; } + ; + +distance_expression returns [ParseNode ret] +@init{ParseNode lhs = null; List<ParseNode> l;} + : i=add_expression {lhs = i;} + (op=(DIST_L2 | DIST_COSINE | DIST_INNER) rhs=add_expression { + l = Arrays.asList(lhs, rhs); + lhs = op.getType() == DIST_L2 ? factory.l2Distance(l) + : op.getType() == DIST_COSINE ? factory.cosineDistance(l) + : factory.innerProductDistance(l); + } + )* + { $ret = lhs; } ; add_expression returns [ParseNode ret] @@ -1514,10 +1527,21 @@ EQ : '=' ; +// Separate the infix distance operators (<=>, <->, <#>) from the less-than operator LT : '<' + ( ('=' '>') => '=' '>' { $type = DIST_COSINE; } + | ('-' '>') => '-' '>' { $type = DIST_L2; } + | ('#' '>') => '#' '>' { $type = DIST_INNER; } + | { $type = LT; } + ) ; +// Token types for the infix distance operators. Only the LT rule above emits these types. +fragment DIST_L2 : '<->' ; +fragment DIST_COSINE : '<=>' ; +fragment DIST_INNER : '<#>' ; + GT : '>' ; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index c4b33a7649..3611dbf1df 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -89,6 +89,7 @@ public enum SQLExceptionCode { MISSING_MAX_LENGTH(207, "22004", "Max length must be specified for type."), NONPOSITIVE_MAX_LENGTH(208, "22006", "Max length must have a positive length for type."), VECTOR_DIMENSION_EXCEEDED(210, "22003", "Vector dimension exceeds the configured maximum."), + VECTOR_DIMENSION_MISMATCH(211, "22000", "Vector dimensions do not match."), DECIMAL_PRECISION_OUT_OF_RANGE(209, "22003", "Decimal precision outside of range. Should be within 1 and " + PDataType.MAX_PRECISION + "."), SERVER_ARITHMETIC_ERROR(212, "22012", "Arithmetic error on server."), diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java index 6b280cf9de..c857a6f2e6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -211,7 +211,11 @@ public enum ExpressionType { RawRowSizeFunction(RawRowSizeFunction.class), RegexpLikeFunction(RegexpLikeFunction.class), ByteBasedRegexpLikeFunction(ByteBasedRegexpLikeFunction.class), - StringBasedRegexpLikeFunction(StringBasedRegexpLikeFunction.class); + StringBasedRegexpLikeFunction(StringBasedRegexpLikeFunction.class), + L2DistanceFunction(L2DistanceFunction.class), + L2DistanceSquaredFunction(L2DistanceSquaredFunction.class), + CosineDistanceFunction(CosineDistanceFunction.class), + InnerProductDistanceFunction(InnerProductDistanceFunction.class); ExpressionType(Class<? extends Expression> clazz) { this.clazz = clazz; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/CosineDistanceFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/CosineDistanceFunction.java new file mode 100644 index 0000000000..9f37db957d --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/CosineDistanceFunction.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.util.List; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.parse.DistanceFunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.Argument; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; + +/** + * Cosine distance: {@code 1 - dot(a, b) / (norm(a) * norm(b))}, in the range 0 to 2. If one or both + * vectors have zero magnitude, the distance is 1.0, which is the distance for orthogonal vectors. + */ +@BuiltInFunction(name = CosineDistanceFunction.NAME, nodeClass = DistanceFunctionParseNode.class, + args = { @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }), + @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }) }) +public class CosineDistanceFunction extends DistanceFunction { + + public static final String NAME = "COSINE_DISTANCE"; + + public CosineDistanceFunction() { + } + + public CosineDistanceFunction(List<Expression> children) { + super(children); + } + + @Override + public String getName() { + return NAME; + } + + @Override + protected double computeFloat(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + return VectorDistanceUtil.cosineDistanceWithBound(a, aOff, b, bOff, dim, bound); + } + + @Override + protected double computeGeneric(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, byte[] b, + int bOff, boolean bIsDouble, SortOrder bOrder, int dim, double bound) { + double dot = 0.0; + double normA = 0.0; + double normB = 0.0; + for (int i = 0; i < dim; i++) { + double ai = element(a, aOff, aIsDouble, aOrder, i); + double bi = element(b, bOff, bIsDouble, bOrder, i); + dot += ai * bi; + normA += ai * ai; + normB += bi * bi; + } + return VectorDistanceUtil.applyBound(ScalarDistanceKernel.cosineDistance(dot, normA, normB), + bound); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceFunction.java new file mode 100644 index 0000000000..c3773de938 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceFunction.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.sql.SQLException; +import java.util.List; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.exception.SQLExceptionInfo; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.schema.IllegalDataException; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; + +/** + * Base class for the vector distance functions. The functions read the packed byte encodings of the + * operands directly and do not decode the elements into heap arrays. Each evaluation writes the + * result into a new buffer, because downstream sort steps keep a reference to each result. A null + * operand gives a null result. Operands with different dimensions cause an + * {@link IllegalDataException}. + */ +public abstract class DistanceFunction extends ScalarFunction { + + /** + * Upper bound for a bounded top-N scan. If the distance of a candidate is more than this bound, + * the result is {@link Double#MAX_VALUE}, and the L2 kernels can stop early. Expression + * serialization does not include this value, but {@link #clone(List)} copies it. + */ + private double distanceUpperBound = Double.MAX_VALUE; + + public DistanceFunction() { + } + + public DistanceFunction(List<Expression> children) { + super(children); + } + + /** + * Makes sure that both arguments are vectors and, if both dimensions are known at compile time, + * that the dimensions are equal. An argument of unknown type passes. If a dimension is not known, + * {@link #evaluate} checks the dimensions for each row. + * @throws SQLException if an argument is not a vector or the two dimensions are different + */ + public static void validateChildren(List<Expression> children) throws SQLException { + for (int i = 0; i < 2; i++) { + PDataType type = children.get(i).getDataType(); + if (type != null && !type.isVectorType()) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.TYPE_MISMATCH) + .setMessage("Distance function argument " + (i + 1) + " must be a vector, but was " + + type.getSqlTypeName()) + .build().buildException(); + } + } + Integer dim1 = children.get(0).getMaxLength(); + Integer dim2 = children.get(1).getMaxLength(); + if (dim1 != null && dim2 != null && !dim1.equals(dim2)) { + throw dimensionMismatch(dim1, dim2); + } + } + + private static SQLException dimensionMismatch(int dim1, int dim2) { + return new SQLExceptionInfo.Builder(SQLExceptionCode.VECTOR_DIMENSION_MISMATCH) + .setMessage(dim1 + " != " + dim2).build().buildException(); + } + + public double getDistanceUpperBound() { + return distanceUpperBound; + } + + public void setDistanceUpperBound(double distanceUpperBound) { + this.distanceUpperBound = distanceUpperBound; + } + + @Override + public PDataType getDataType() { + return PDouble.INSTANCE; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + Expression child1 = children.get(0); + if (!child1.evaluate(tuple, ptr)) { + return false; + } + if (ptr.getLength() == 0) { + return true; + } + byte[] buf1 = ptr.get(); + int off1 = ptr.getOffset(); + boolean isDouble1 = isDouble(child1); + int dim1 = ptr.getLength() / (isDouble1 ? Bytes.SIZEOF_DOUBLE : Bytes.SIZEOF_FLOAT); + + // Keep the first operand buffer, because the second child evaluation resets ptr + Expression child2 = children.get(1); + if (!child2.evaluate(tuple, ptr)) { + return false; + } + if (ptr.getLength() == 0) { + return true; + } + byte[] buf2 = ptr.get(); + int off2 = ptr.getOffset(); + boolean isDouble2 = isDouble(child2); + int dim2 = ptr.getLength() / (isDouble2 ? Bytes.SIZEOF_DOUBLE : Bytes.SIZEOF_FLOAT); + if (dim1 != dim2) { + throw new IllegalDataException(dimensionMismatch(dim1, dim2)); + } + + double distance; + if ( + !isDouble1 && !isDouble2 && child1.getSortOrder() == SortOrder.ASC + && child2.getSortOrder() == SortOrder.ASC + ) { + distance = computeFloat(buf1, off1, buf2, off2, dim1, distanceUpperBound); + } else { + distance = computeGeneric(buf1, off1, isDouble1, child1.getSortOrder(), buf2, off2, isDouble2, + child2.getSortOrder(), dim1, distanceUpperBound); + } + byte[] out = new byte[Bytes.SIZEOF_DOUBLE]; + PDouble.INSTANCE.getCodec().encodeDouble(distance, out, 0); + ptr.set(out); + return true; + } + + private static boolean isDouble(Expression child) { + return child.getDataType() == PVectorDouble.INSTANCE; + } + + /** + * Computes the distance between two float vectors in ASC sort order with the + * {@link VectorDistanceUtil} kernels, which can use SIMD. Returns {@link Double#MAX_VALUE} if the + * distance is more than {@code bound}. + */ + protected abstract double computeFloat(byte[] a, int aOff, byte[] b, int bOff, int dim, + double bound); + + /** + * Computes the distance for all other combinations of element precision and sort order. Returns + * {@link Double#MAX_VALUE} if the distance is more than {@code bound}. + */ + protected abstract double computeGeneric(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, + byte[] b, int bOff, boolean bIsDouble, SortOrder bOrder, int dim, double bound); + + /** Decodes the element at index {@code i} for the given precision and sort order. */ + protected static double element(byte[] buf, int off, boolean isDouble, SortOrder order, int i) { + return isDouble + ? PVectorDouble.readElement(buf, off, i, order) + : PVectorFloat.readElement(buf, off, i, order); + } + + @Override + public DistanceFunction clone(List<Expression> children) { + try { + DistanceFunction clone = + (DistanceFunction) getClass().getConstructor(List.class).newInstance(children); + clone.setDistanceUpperBound(this.distanceUpperBound); + return clone; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceKernel.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceKernel.java new file mode 100644 index 0000000000..8fd7508e30 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DistanceKernel.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +/** + * Default query time distance kernels for packed float vectors. This class calls + * {@link ScalarDistanceKernel}. On Java 21, a multi-release variant replaces this class and calls + * the Panama Vector API kernels if the {@code jdk.incubator.vector} module is available. + */ +final class DistanceKernel { + + private DistanceKernel() { + } + + /** Returns true if query time distances use SIMD kernels. This variant always returns false. */ + static boolean isSimd() { + return false; + } + + static double l2DistanceSquared(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + return ScalarDistanceKernel.l2DistanceSquared(a, aOff, b, bOff, dim, bound); + } + + static double dotProduct(byte[] a, int aOff, byte[] b, int bOff, int dim) { + return ScalarDistanceKernel.dotProduct(a, aOff, b, bOff, dim); + } + + static double cosineDistance(byte[] a, int aOff, byte[] b, int bOff, int dim) { + return ScalarDistanceKernel.cosineDistance(a, aOff, b, bOff, dim); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/InnerProductDistanceFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/InnerProductDistanceFunction.java new file mode 100644 index 0000000000..5202e3678e --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/InnerProductDistanceFunction.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.util.List; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.parse.DistanceFunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.Argument; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; + +/** + * Negative inner product distance: {@code -dot(a, b)}. A larger inner product gives a smaller + * distance, so an ascending sort puts the most similar vectors first. + */ +@BuiltInFunction(name = InnerProductDistanceFunction.NAME, + nodeClass = DistanceFunctionParseNode.class, + args = { @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }), + @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }) }) +public class InnerProductDistanceFunction extends DistanceFunction { + + public static final String NAME = "INNER_PRODUCT"; + + public InnerProductDistanceFunction() { + } + + public InnerProductDistanceFunction(List<Expression> children) { + super(children); + } + + @Override + public String getName() { + return NAME; + } + + @Override + protected double computeFloat(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + return VectorDistanceUtil.innerProductDistanceWithBound(a, aOff, b, bOff, dim, bound); + } + + @Override + protected double computeGeneric(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, byte[] b, + int bOff, boolean bIsDouble, SortOrder bOrder, int dim, double bound) { + double dot = 0.0; + for (int i = 0; i < dim; i++) { + dot += element(a, aOff, aIsDouble, aOrder, i) * element(b, bOff, bIsDouble, bOrder, i); + } + return VectorDistanceUtil.applyBound(-dot, bound); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceFunction.java new file mode 100644 index 0000000000..afcb817e5a --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceFunction.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.util.List; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.parse.DistanceFunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.Argument; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; + +/** Euclidean distance: {@code sqrt(sum((a[i] - b[i])^2))}. */ +@BuiltInFunction(name = L2DistanceFunction.NAME, nodeClass = DistanceFunctionParseNode.class, + args = { @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }), + @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }) }) +public class L2DistanceFunction extends DistanceFunction { + + public static final String NAME = "L2_DISTANCE"; + + public L2DistanceFunction() { + } + + public L2DistanceFunction(List<Expression> children) { + super(children); + } + + @Override + public String getName() { + return NAME; + } + + @Override + protected double computeFloat(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + return VectorDistanceUtil.l2DistanceWithBound(a, aOff, b, bOff, dim, bound); + } + + @Override + protected double computeGeneric(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, byte[] b, + int bOff, boolean bIsDouble, SortOrder bOrder, int dim, double bound) { + double boundSq = bound == Double.MAX_VALUE ? Double.MAX_VALUE : bound * bound; + double sq = L2DistanceSquaredFunction.squared(a, aOff, aIsDouble, aOrder, b, bOff, bIsDouble, + bOrder, dim, boundSq); + return sq == Double.MAX_VALUE + ? Double.MAX_VALUE + : VectorDistanceUtil.applyBound(Math.sqrt(sq), bound); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceSquaredFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceSquaredFunction.java new file mode 100644 index 0000000000..b01753f6de --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/L2DistanceSquaredFunction.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.util.List; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.parse.DistanceFunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.Argument; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; + +/** + * Squared Euclidean distance: {@code sum((a[i] - b[i])^2)}. It gives the same rank order as + * {@code L2_DISTANCE} without the square root. + */ +@BuiltInFunction(name = L2DistanceSquaredFunction.NAME, nodeClass = DistanceFunctionParseNode.class, + args = { @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }), + @Argument(allowedTypes = { PVectorFloat.class, PVectorDouble.class }) }) +public class L2DistanceSquaredFunction extends DistanceFunction { + + public static final String NAME = "L2_DISTANCE_SQUARED"; + + public L2DistanceSquaredFunction() { + } + + public L2DistanceSquaredFunction(List<Expression> children) { + super(children); + } + + @Override + public String getName() { + return NAME; + } + + @Override + protected double computeFloat(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + return VectorDistanceUtil.l2DistanceSquaredWithBound(a, aOff, b, bOff, dim, bound); + } + + @Override + protected double computeGeneric(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, byte[] b, + int bOff, boolean bIsDouble, SortOrder bOrder, int dim, double bound) { + return squared(a, aOff, aIsDouble, aOrder, b, bOff, bIsDouble, bOrder, dim, bound); + } + + static double squared(byte[] a, int aOff, boolean aIsDouble, SortOrder aOrder, byte[] b, int bOff, + boolean bIsDouble, SortOrder bOrder, int dim, double bound) { + double sum = 0.0; + for (int i = 0; i < dim; i++) { + double diff = element(a, aOff, aIsDouble, aOrder, i) - element(b, bOff, bIsDouble, bOrder, i); + sum += diff * diff; + if (sum > bound) { + return Double.MAX_VALUE; + } + } + return sum; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/ScalarDistanceKernel.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/ScalarDistanceKernel.java new file mode 100644 index 0000000000..64ac24423e --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/ScalarDistanceKernel.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Reference scalar distance kernels for packed float bytes and {@code float[]} arrays. The kernels + * add the terms in double precision and in element order, so the results are the same on all + * platforms. + */ +final class ScalarDistanceKernel { + + private ScalarDistanceKernel() { + } + + private static float read(byte[] buf, int off, int i) { + return Bytes.toFloat(buf, off + i * Bytes.SIZEOF_FLOAT); + } + + /** + * Computes the squared Euclidean distance. If the partial sum becomes more than {@code bound}, + * the method stops and returns {@link Double#MAX_VALUE}. + */ + static double l2DistanceSquared(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + double sum = 0.0; + for (int i = 0; i < dim; i++) { + double diff = read(a, aOff, i) - read(b, bOff, i); + sum += diff * diff; + if (sum > bound) { + return Double.MAX_VALUE; + } + } + return sum; + } + + static double l2DistanceSquared(float[] a, float[] b, int dim, double bound) { + double sum = 0.0; + for (int i = 0; i < dim; i++) { + double diff = a[i] - b[i]; + sum += diff * diff; + if (sum > bound) { + return Double.MAX_VALUE; + } + } + return sum; + } + + static double dotProduct(byte[] a, int aOff, byte[] b, int bOff, int dim) { + double dot = 0.0; + for (int i = 0; i < dim; i++) { + dot += (double) read(a, aOff, i) * read(b, bOff, i); + } + return dot; + } + + static double dotProduct(float[] a, float[] b, int dim) { + double dot = 0.0; + for (int i = 0; i < dim; i++) { + dot += (double) a[i] * b[i]; + } + return dot; + } + + static double cosineDistance(byte[] a, int aOff, byte[] b, int bOff, int dim) { + double dot = 0.0; + double normA = 0.0; + double normB = 0.0; + for (int i = 0; i < dim; i++) { + double ai = read(a, aOff, i); + double bi = read(b, bOff, i); + dot += ai * bi; + normA += ai * ai; + normB += bi * bi; + } + return cosineDistance(dot, normA, normB); + } + + static double cosineDistance(float[] a, float[] b, int dim) { + double dot = 0.0; + double normA = 0.0; + double normB = 0.0; + for (int i = 0; i < dim; i++) { + double ai = a[i]; + double bi = b[i]; + dot += ai * bi; + normA += ai * ai; + normB += bi * bi; + } + return cosineDistance(dot, normA, normB); + } + + /** + * Computes the cosine distance from a dot product and two squared norms. If an operand has zero + * magnitude, the result is 1.0, which is the distance for orthogonal vectors. The similarity is + * clamped to the range -1 to 1, so rounding errors cannot move the result out of 0 to 2. + */ + static double cosineDistance(double dot, double normA, double normB) { + double denom = Math.sqrt(normA) * Math.sqrt(normB); + if (denom == 0.0) { + return 1.0; + } + double similarity = Math.max(-1.0, Math.min(1.0, dot / denom)); + return 1.0 - similarity; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/VectorDistanceUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/VectorDistanceUtil.java new file mode 100644 index 0000000000..90b3f8578e --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/VectorDistanceUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +/** + * Vector distance methods for packed float bytes and {@code float[]} arrays. + * <p> + * The bounded methods are for query time. They call {@link DistanceKernel}, which can use SIMD. If + * the distance is more than the bound, they return {@link Double#MAX_VALUE}, and the L2 methods can + * stop early. Use {@link Double#MAX_VALUE} as the bound for no limit. + * <p> + * The unbounded scalar methods never use SIMD, so their results are the same on all platforms. + * Centroid training and centroid assignment use these methods for this reason. + */ +public final class VectorDistanceUtil { + + private VectorDistanceUtil() { + } + + /** Returns true if the query time distance kernels use SIMD in this JVM. */ + public static boolean isSimdEnabled() { + return DistanceKernel.isSimd(); + } + + public static double scalarL2DistanceSquared(float[] a, float[] b, int dim) { + return ScalarDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE); + } + + public static double scalarInnerProductDistance(float[] a, float[] b, int dim) { + return -ScalarDistanceKernel.dotProduct(a, b, dim); + } + + public static double scalarCosineDistance(float[] a, float[] b, int dim) { + return ScalarDistanceKernel.cosineDistance(a, b, dim); + } + + public static double l2DistanceSquaredWithBound(byte[] a, int aOff, byte[] b, int bOff, int dim, + double bound) { + return DistanceKernel.l2DistanceSquared(a, aOff, b, bOff, dim, bound); + } + + public static double l2DistanceWithBound(byte[] a, int aOff, byte[] b, int bOff, int dim, + double bound) { + double boundSq = bound == Double.MAX_VALUE ? Double.MAX_VALUE : bound * bound; + double sq = DistanceKernel.l2DistanceSquared(a, aOff, b, bOff, dim, boundSq); + return sq == Double.MAX_VALUE ? Double.MAX_VALUE : applyBound(Math.sqrt(sq), bound); + } + + public static double innerProductDistanceWithBound(byte[] a, int aOff, byte[] b, int bOff, + int dim, double bound) { + return applyBound(-DistanceKernel.dotProduct(a, aOff, b, bOff, dim), bound); + } + + public static double cosineDistanceWithBound(byte[] a, int aOff, byte[] b, int bOff, int dim, + double bound) { + return applyBound(DistanceKernel.cosineDistance(a, aOff, b, bOff, dim), bound); + } + + static double applyBound(double distance, double bound) { + return distance > bound ? Double.MAX_VALUE : distance; + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DistanceFunctionParseNode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DistanceFunctionParseNode.java new file mode 100644 index 0000000000..4437a8c57b --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DistanceFunctionParseNode.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import java.sql.SQLException; +import java.util.List; +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.function.DistanceFunction; + +/** + * Parse node for the vector distance functions. It converts an array argument to the vector type of + * the other argument, or to a float vector if neither argument is a vector. Then it makes sure that + * the types and dimensions of the two vectors are compatible, before it creates the function. + */ +public class DistanceFunctionParseNode extends FunctionParseNode { + + public DistanceFunctionParseNode(String name, List<ParseNode> children, + BuiltInFunctionInfo info) { + super(name, children, info); + } + + @Override + public Expression create(List<Expression> children, StatementContext context) + throws SQLException { + DistanceFunction.validateChildren(children); + return super.create(children, context); + } +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index 2a36dc4035..da08d77a76 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -33,11 +33,14 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.ExpressionType; import org.apache.phoenix.expression.function.AvgAggregateFunction; +import org.apache.phoenix.expression.function.CosineDistanceFunction; import org.apache.phoenix.expression.function.CountAggregateFunction; import org.apache.phoenix.expression.function.CurrentDateFunction; import org.apache.phoenix.expression.function.CurrentTimeFunction; import org.apache.phoenix.expression.function.DistinctCountAggregateFunction; import org.apache.phoenix.expression.function.FunctionExpression; +import org.apache.phoenix.expression.function.InnerProductDistanceFunction; +import org.apache.phoenix.expression.function.L2DistanceFunction; import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunctionInfo; import org.apache.phoenix.parse.JoinTableNode.JoinType; @@ -243,6 +246,18 @@ public class ParseNodeFactory { return new ModulusParseNode(children); } + public FunctionParseNode l2Distance(List<ParseNode> children) { + return function(L2DistanceFunction.NAME, children); + } + + public FunctionParseNode cosineDistance(List<ParseNode> children) { + return function(CosineDistanceFunction.NAME, children); + } + + public FunctionParseNode innerProductDistance(List<ParseNode> children) { + return function(InnerProductDistanceFunction.NAME, children); + } + public AndParseNode and(List<ParseNode> children) { return new AndParseNode(children); } diff --git a/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/DistanceKernel.java b/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/DistanceKernel.java new file mode 100644 index 0000000000..6fff9f6cb8 --- /dev/null +++ b/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/DistanceKernel.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Java 21 multi-release implementation of {@code DistanceKernel}. Dispatches to + * {@link PanamaDistanceKernel} when {@code jdk.incubator.vector} is available, falling back to + * {@link ScalarDistanceKernel}. + * <p> + * Decodes packed big-endian float vectors into thread local float buffers prior to SIMD execution + * to satisfy Vector API buffer alignment and endianness requirements. + */ +final class DistanceKernel { + + private static final boolean SIMD = isPanamaAvailable(); + + private static final ThreadLocal<float[][]> SCRATCH = + ThreadLocal.withInitial(() -> new float[][] { new float[0], new float[0] }); + + private DistanceKernel() { + } + + static boolean isSimd() { + return SIMD; + } + + private static boolean isPanamaAvailable() { + try { + Class.forName("jdk.incubator.vector.FloatVector"); + return PanamaDistanceKernel.dotProduct(new float[] { 1f }, new float[] { 1f }, 1) == 1.0; + } catch (Throwable t) { + return false; + } + } + + private static float[] decode(int slot, byte[] buf, int off, int dim) { + float[][] scratch = SCRATCH.get(); + float[] out = scratch[slot]; + if (out.length < dim) { + out = new float[dim]; + scratch[slot] = out; + } + for (int i = 0; i < dim; i++) { + out[i] = Bytes.toFloat(buf, off + i * Bytes.SIZEOF_FLOAT); + } + return out; + } + + static double l2DistanceSquared(byte[] a, int aOff, byte[] b, int bOff, int dim, double bound) { + if (!SIMD) { + return ScalarDistanceKernel.l2DistanceSquared(a, aOff, b, bOff, dim, bound); + } + return PanamaDistanceKernel.l2DistanceSquared(decode(0, a, aOff, dim), decode(1, b, bOff, dim), + dim, bound); + } + + static double dotProduct(byte[] a, int aOff, byte[] b, int bOff, int dim) { + if (!SIMD) { + return ScalarDistanceKernel.dotProduct(a, aOff, b, bOff, dim); + } + return PanamaDistanceKernel.dotProduct(decode(0, a, aOff, dim), decode(1, b, bOff, dim), dim); + } + + static double cosineDistance(byte[] a, int aOff, byte[] b, int bOff, int dim) { + if (!SIMD) { + return ScalarDistanceKernel.cosineDistance(a, aOff, b, bOff, dim); + } + return PanamaDistanceKernel.cosineDistance(decode(0, a, aOff, dim), decode(1, b, bOff, dim), + dim); + } +} diff --git a/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/PanamaDistanceKernel.java b/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/PanamaDistanceKernel.java new file mode 100644 index 0000000000..386f1a4f86 --- /dev/null +++ b/phoenix-core-client/src/main/java21/org/apache/phoenix/expression/function/PanamaDistanceKernel.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import jdk.incubator.vector.FloatVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +/** + * SIMD distance kernels leveraging the Panama Vector API over {@code float[]} operands. + * <p> + * Accumulates intermediate lane sums in single precision across fixed size blocks of + * {@link #BLOCK_STRIDES} vector strides before reducing to double precision. Block level reduction + * bounds floating point accumulation drift relative to {@link ScalarDistanceKernel} while enabling + * periodic upper bound pruning without per-stride reduction overhead. + */ +final class PanamaDistanceKernel { + + private static final VectorSpecies<Float> SPECIES = FloatVector.SPECIES_PREFERRED; + static final int BLOCK_STRIDES = 8; + + private PanamaDistanceKernel() { + } + + static double l2DistanceSquared(float[] a, float[] b, int dim, double bound) { + int step = SPECIES.length(); + int upper = SPECIES.loopBound(dim); + int blockLen = step * BLOCK_STRIDES; + double total = 0.0; + int i = 0; + while (i < upper) { + int blockEnd = Math.min(upper, i + blockLen); + FloatVector acc = FloatVector.zero(SPECIES); + for (; i < blockEnd; i += step) { + FloatVector diff = + FloatVector.fromArray(SPECIES, a, i).sub(FloatVector.fromArray(SPECIES, b, i)); + acc = diff.fma(diff, acc); + } + total += acc.reduceLanes(VectorOperators.ADD); + if (total > bound) { + return Double.MAX_VALUE; + } + } + for (; i < dim; i++) { + double diff = a[i] - b[i]; + total += diff * diff; + if (total > bound) { + return Double.MAX_VALUE; + } + } + return total; + } + + static double dotProduct(float[] a, float[] b, int dim) { + int step = SPECIES.length(); + int upper = SPECIES.loopBound(dim); + int blockLen = step * BLOCK_STRIDES; + double total = 0.0; + int i = 0; + while (i < upper) { + int blockEnd = Math.min(upper, i + blockLen); + FloatVector acc = FloatVector.zero(SPECIES); + for (; i < blockEnd; i += step) { + acc = FloatVector.fromArray(SPECIES, a, i).fma(FloatVector.fromArray(SPECIES, b, i), acc); + } + total += acc.reduceLanes(VectorOperators.ADD); + } + for (; i < dim; i++) { + total += (double) a[i] * b[i]; + } + return total; + } + + static double cosineDistance(float[] a, float[] b, int dim) { + int step = SPECIES.length(); + int upper = SPECIES.loopBound(dim); + int blockLen = step * BLOCK_STRIDES; + double dot = 0.0; + double normA = 0.0; + double normB = 0.0; + int i = 0; + while (i < upper) { + int blockEnd = Math.min(upper, i + blockLen); + FloatVector vDot = FloatVector.zero(SPECIES); + FloatVector vNormA = FloatVector.zero(SPECIES); + FloatVector vNormB = FloatVector.zero(SPECIES); + for (; i < blockEnd; i += step) { + FloatVector va = FloatVector.fromArray(SPECIES, a, i); + FloatVector vb = FloatVector.fromArray(SPECIES, b, i); + vDot = va.fma(vb, vDot); + vNormA = va.fma(va, vNormA); + vNormB = vb.fma(vb, vNormB); + } + dot += vDot.reduceLanes(VectorOperators.ADD); + normA += vNormA.reduceLanes(VectorOperators.ADD); + normB += vNormB.reduceLanes(VectorOperators.ADD); + } + for (; i < dim; i++) { + double ai = a[i]; + double bi = b[i]; + dot += ai * bi; + normA += ai * ai; + normB += bi * bi; + } + return ScalarDistanceKernel.cosineDistance(dot, normA, normB); + } +} diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml index a69db4b597..5de59b9072 100644 --- a/phoenix-core/pom.xml +++ b/phoenix-core/pom.xml @@ -518,4 +518,39 @@ </plugin> </plugins> </build> + <profiles> + <profile> + <!-- Compile the Java 21 distance kernels and tests into the test classes. The test suite + then runs with the SIMD kernels. --> + <id>java21</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <executions> + <execution> + <id>test-compile-java21</id> + <goals> + <goal>testCompile</goal> + </goals> + <phase>test-compile</phase> + <configuration> + <release>21</release> + <compileSourceRoots> + <compileSourceRoot>${project.basedir}/../phoenix-core-client/src/main/java21</compileSourceRoot> + <compileSourceRoot>${project.basedir}/src/test/java21</compileSourceRoot> + </compileSourceRoots> + <compilerArgs> + <arg>--add-modules</arg> + <arg>jdk.incubator.vector</arg> + </compilerArgs> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/function/DistanceFunctionsTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/DistanceFunctionsTest.java new file mode 100644 index 0000000000..89aaa3a5c8 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/DistanceFunctionsTest.java @@ -0,0 +1,531 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.compile.QueryPlan; +import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.ExpressionType; +import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.parse.ColumnParseNode; +import org.apache.phoenix.parse.DistanceFunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunctionInfo; +import org.apache.phoenix.parse.ParseNode; +import org.apache.phoenix.parse.ParseNodeFactory; +import org.apache.phoenix.parse.SQLParser; +import org.apache.phoenix.parse.SelectStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.schema.IllegalDataException; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; +import org.junit.Test; + +import org.apache.phoenix.thirdparty.com.google.common.collect.Multimap; + +public class DistanceFunctionsTest extends BaseConnectionlessQueryTest { + + private static final double DELTA = 1e-6; + + private static <T extends DistanceFunction> T withBound(T function, double bound) { + function.setDistanceUpperBound(bound); + return function; + } + + private static BuiltInFunctionInfo lookup(String name) { + Collection<BuiltInFunctionInfo> infos = ParseNodeFactory.getBuiltInFunctionMultimap().get(name); + return infos.isEmpty() ? null : infos.iterator().next(); + } + + private static void assertDistanceNode(String name, ParseNode node) { + assertTrue("Expected DistanceFunctionParseNode, got " + node.getClass().getSimpleName(), + node instanceof DistanceFunctionParseNode); + assertEquals(name, ((FunctionParseNode) node).getName()); + } + + private static double evaluateDistance(DistanceFunction function) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + boolean evaluated = function.evaluate(null, ptr); + assertTrue("Distance function should evaluate successfully", evaluated); + assertNotNull("Pointer should not be null", ptr.get()); + Object obj = PDouble.INSTANCE.toObject(ptr); + assertNotNull("Decoded distance should not be null", obj); + return ((Number) obj).doubleValue(); + } + + @Test + public void testL2DistanceCorrectness() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE); + L2DistanceFunction func = new L2DistanceFunction(Arrays.asList(v1, v2)); + + double dist = evaluateDistance(func); + assertEquals(5.0, dist, DELTA); + } + + @Test + public void testL2DistanceSquaredCorrectness() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE); + L2DistanceSquaredFunction func = new L2DistanceSquaredFunction(Arrays.asList(v1, v2)); + + double dist = evaluateDistance(func); + assertEquals(25.0, dist, DELTA); + } + + @Test + public void testCosineDistanceOrthogonalVectors() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 1.0f, 0.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 0.0f, 1.0f }, PVectorFloat.INSTANCE); + CosineDistanceFunction func = new CosineDistanceFunction(Arrays.asList(v1, v2)); + + double dist = evaluateDistance(func); + assertEquals(1.0, dist, DELTA); + } + + @Test + public void testCosineDistanceIdenticalAndOpposite() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f, 3.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f, 3.0f }, PVectorFloat.INSTANCE); + CosineDistanceFunction funcIdentical = new CosineDistanceFunction(Arrays.asList(v1, v2)); + assertEquals(0.0, evaluateDistance(funcIdentical), DELTA); + + Expression vOpposite = + LiteralExpression.newConstant(new float[] { -1.0f, -2.0f, -3.0f }, PVectorFloat.INSTANCE); + CosineDistanceFunction funcOpposite = new CosineDistanceFunction(Arrays.asList(v1, vOpposite)); + assertEquals(2.0, evaluateDistance(funcOpposite), DELTA); + } + + @Test + public void testCosineDistanceZeroVector() throws Exception { + Expression vZero = + LiteralExpression.newConstant(new float[] { 0.0f, 0.0f }, PVectorFloat.INSTANCE); + Expression v1 = + LiteralExpression.newConstant(new float[] { 1.0f, 0.0f }, PVectorFloat.INSTANCE); + CosineDistanceFunction func = new CosineDistanceFunction(Arrays.asList(vZero, v1)); + assertEquals(1.0, evaluateDistance(func), DELTA); + } + + @Test + public void testInnerProductDistance() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 2.0f, 3.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 4.0f, 5.0f }, PVectorFloat.INSTANCE); + InnerProductDistanceFunction func = new InnerProductDistanceFunction(Arrays.asList(v1, v2)); + + double dist = evaluateDistance(func); + assertEquals(-23.0, dist, DELTA); + } + + @Test + public void testVectorDoubleSupport() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new double[] { 3.0, 0.0 }, PVectorDouble.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new double[] { 0.0, 4.0 }, PVectorDouble.INSTANCE); + + L2DistanceFunction l2 = new L2DistanceFunction(Arrays.asList(v1, v2)); + assertEquals(5.0, evaluateDistance(l2), DELTA); + + L2DistanceSquaredFunction l2Sq = new L2DistanceSquaredFunction(Arrays.asList(v1, v2)); + assertEquals(25.0, evaluateDistance(l2Sq), DELTA); + + Expression d1 = + LiteralExpression.newConstant(new double[] { 2.0, 3.0 }, PVectorDouble.INSTANCE); + Expression d2 = + LiteralExpression.newConstant(new double[] { 4.0, 5.0 }, PVectorDouble.INSTANCE); + InnerProductDistanceFunction ip = new InnerProductDistanceFunction(Arrays.asList(d1, d2)); + assertEquals(-23.0, evaluateDistance(ip), DELTA); + + Expression o1 = + LiteralExpression.newConstant(new double[] { 1.0, 0.0 }, PVectorDouble.INSTANCE); + Expression o2 = + LiteralExpression.newConstant(new double[] { 0.0, 1.0 }, PVectorDouble.INSTANCE); + CosineDistanceFunction cosine = new CosineDistanceFunction(Arrays.asList(o1, o2)); + assertEquals(1.0, evaluateDistance(cosine), DELTA); + } + + @Test + public void testDescendingSortOrder() throws Exception { + Expression v1 = LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE, + SortOrder.DESC); + Expression v2 = LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE, + SortOrder.DESC); + + L2DistanceFunction l2 = new L2DistanceFunction(Arrays.asList(v1, v2)); + assertEquals(5.0, evaluateDistance(l2), DELTA); + + Expression v3 = LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE, + SortOrder.ASC); + L2DistanceFunction l2Mixed = new L2DistanceFunction(Arrays.asList(v1, v3)); + assertEquals(5.0, evaluateDistance(l2Mixed), DELTA); + } + + @Test + public void testDimensionMismatchCompileTime() throws Exception { + Expression v2 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f }, PVectorFloat.INSTANCE); + Expression v3 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f, 3.0f }, PVectorFloat.INSTANCE); + try { + DistanceFunction.validateChildren(Arrays.asList(v2, v3)); + fail("Expected SQLException for dimension mismatch"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.VECTOR_DIMENSION_MISMATCH.getErrorCode(), e.getErrorCode()); + } + } + + @Test + public void testDimensionMismatchAtEvaluation() throws Exception { + // Operands with different dimensions must fail at row evaluation + Expression v2 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f }, PVectorFloat.INSTANCE); + Expression v3 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f, 3.0f }, PVectorFloat.INSTANCE); + for (DistanceFunction func : new DistanceFunction[] { + new L2DistanceFunction(Arrays.asList(v2, v3)), + new L2DistanceSquaredFunction(Arrays.asList(v2, v3)), + new CosineDistanceFunction(Arrays.asList(v2, v3)), + new InnerProductDistanceFunction(Arrays.asList(v2, v3)) }) { + try { + func.evaluate(null, new ImmutableBytesWritable()); + fail("Expected IllegalDataException for " + func.getName()); + } catch (IllegalDataException e) { + assertEquals(SQLExceptionCode.VECTOR_DIMENSION_MISMATCH.getErrorCode(), + ((SQLException) e.getCause()).getErrorCode()); + } + } + } + + @Test + public void testNullPropagation() throws Exception { + Expression nullVec = LiteralExpression.newConstant(null, PVectorFloat.INSTANCE); + Expression validVec = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f }, PVectorFloat.INSTANCE); + + DistanceFunction[] funcs = + new DistanceFunction[] { new L2DistanceFunction(Arrays.asList(nullVec, validVec)), + new L2DistanceFunction(Arrays.asList(validVec, nullVec)), + new L2DistanceFunction(Arrays.asList(nullVec, nullVec)), + new L2DistanceSquaredFunction(Arrays.asList(nullVec, validVec)), + new CosineDistanceFunction(Arrays.asList(nullVec, validVec)), + new InnerProductDistanceFunction(Arrays.asList(nullVec, validVec)) }; + + for (DistanceFunction func : funcs) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(new byte[] { 1 }); + assertTrue("A null operand is evaluable for " + func.getName(), func.evaluate(null, ptr)); + assertEquals("Expected SQL NULL for " + func.getName(), 0, ptr.getLength()); + } + } + + @Test + public void testEarlyTerminationUpperBound() throws Exception { + Expression v1 = + LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE); + Expression v2 = + LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE); + + // A squared L2 distance above the upper bound must return Double.MAX_VALUE + L2DistanceSquaredFunction l2SqExceeded = + withBound(new L2DistanceSquaredFunction(Arrays.asList(v1, v2)), 10.0); + assertEquals(Double.MAX_VALUE, evaluateDistance(l2SqExceeded), DELTA); + + L2DistanceSquaredFunction l2SqMet = + withBound(new L2DistanceSquaredFunction(Arrays.asList(v1, v2)), 30.0); + assertEquals(25.0, evaluateDistance(l2SqMet), DELTA); + + // An L2 distance above the upper bound must return Double.MAX_VALUE + L2DistanceFunction l2Exceeded = withBound(new L2DistanceFunction(Arrays.asList(v1, v2)), 4.0); + assertEquals(Double.MAX_VALUE, evaluateDistance(l2Exceeded), DELTA); + + L2DistanceFunction l2Met = withBound(new L2DistanceFunction(Arrays.asList(v1, v2)), 6.0); + assertEquals(5.0, evaluateDistance(l2Met), DELTA); + } + + @Test + public void testParseNodeCreation() throws Exception { + Expression v2 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f }, PVectorFloat.INSTANCE); + Expression v3 = + LiteralExpression.newConstant(new float[] { 1.0f, 2.0f, 3.0f }, PVectorFloat.INSTANCE); + Expression v2b = + LiteralExpression.newConstant(new float[] { 3.0f, 4.0f }, PVectorFloat.INSTANCE); + List<ParseNode> dummyNodes = Collections.emptyList(); + for (int i = 0; i < DISTANCE_FUNCTION_NAMES.length; i++) { + String name = DISTANCE_FUNCTION_NAMES[i]; + DistanceFunctionParseNode node = + new DistanceFunctionParseNode(name, dummyNodes, lookup(name)); + try { + node.create(Arrays.asList(v2, v3), null); + fail("Expected SQLException on dimension mismatch in parse node for " + name); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.VECTOR_DIMENSION_MISMATCH.getErrorCode(), e.getErrorCode()); + } + assertTrue( + DISTANCE_FUNCTION_CLASSES[i].isInstance(node.create(Arrays.asList(v2, v2b), null))); + } + } + + @Test + public void testExpressionTypeFullSerializationRoundTrip() throws Exception { + // Each function must survive a round trip through expression serialization + LiteralExpression child1 = + LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE, 2, null); + LiteralExpression child2 = + LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE, 2, null); + + double[] expectedValues = { 5.0, 25.0, 1.0, 0.0 }; + List<DistanceFunction> functions = + Arrays.asList(new L2DistanceFunction(Arrays.asList(child1, child2)), + new L2DistanceSquaredFunction(Arrays.asList(child1, child2)), + new CosineDistanceFunction(Arrays.asList(child1, child2)), + new InnerProductDistanceFunction(Arrays.asList(child1, child2))); + + for (int i = 0; i < functions.size(); i++) { + DistanceFunction func = functions.get(i); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + WritableUtils.writeVInt(out, ExpressionType.valueOf(func).ordinal()); + func.write(out); + out.close(); + + DataInputStream in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); + int ordinal = WritableUtils.readVInt(in); + Expression deserialized = ExpressionType.values()[ordinal].newInstance(); + deserialized.readFields(in); + + assertTrue(func.getClass().isInstance(deserialized)); + DistanceFunction deserializedDistance = (DistanceFunction) deserialized; + assertEquals(Double.MAX_VALUE, deserializedDistance.getDistanceUpperBound(), 0.0); + assertEquals(2, deserializedDistance.getChildren().size()); + + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + assertTrue(deserializedDistance.evaluate(null, ptr)); + double val = (Double) PDouble.INSTANCE.toObject(ptr); + assertEquals(func.getClass().getSimpleName() + " returned unexpected distance", + expectedValues[i], val, DELTA); + } + } + + private static final String[] DISTANCE_FUNCTION_NAMES = + { L2DistanceFunction.NAME, L2DistanceSquaredFunction.NAME, CosineDistanceFunction.NAME, + InnerProductDistanceFunction.NAME }; + + private static final Class<?>[] DISTANCE_FUNCTION_CLASSES = + { L2DistanceFunction.class, L2DistanceSquaredFunction.class, CosineDistanceFunction.class, + InnerProductDistanceFunction.class }; + + @Test + public void testFunctionLookupByName() { + for (int i = 0; i < DISTANCE_FUNCTION_NAMES.length; i++) { + String name = DISTANCE_FUNCTION_NAMES[i]; + BuiltInFunctionInfo info = lookup(name); + assertNotNull("Function lookup failed for: " + name, info); + assertEquals(name, info.getName()); + assertEquals(2, info.getRequiredArgCount()); + assertEquals(2, info.getArgs().length); + assertEquals(DistanceFunctionParseNode.class, info.getNodeCtor().getDeclaringClass()); + assertEquals(DISTANCE_FUNCTION_CLASSES[i], info.getFunc()); + + // Each argument must accept both float and double vectors + for (FunctionParseNode.BuiltInFunctionArgInfo arg : info.getArgs()) { + List<Class<?>> allowedTypes = Arrays.asList(arg.getAllowedTypes()); + assertTrue(name + " arg should allow PVectorFloat", + allowedTypes.contains(PVectorFloat.class)); + assertTrue(name + " arg should allow PVectorDouble", + allowedTypes.contains(PVectorDouble.class)); + } + } + } + + @Test + public void testFunctionLookupWithChildrenList() { + ParseNode c1 = new ColumnParseNode(null, "V1", null); + ParseNode c2 = new ColumnParseNode(null, "V2", null); + List<ParseNode> children2 = Arrays.asList(c1, c2); + List<ParseNode> children1 = Arrays.asList(c1); + + for (String name : DISTANCE_FUNCTION_NAMES) { + BuiltInFunctionInfo info = ParseNodeFactory.get(name, children2); + assertNotNull("Lookup with 2 children failed for: " + name, info); + assertEquals(name, info.getName()); + + assertNull("Lookup with 1 child should be null for: " + name, + ParseNodeFactory.get(name, children1)); + } + } + + @Test + public void testFunctionUniquenessAndNoCollisions() { + Multimap<String, BuiltInFunctionInfo> multimap = ParseNodeFactory.getBuiltInFunctionMultimap(); + + for (String name : DISTANCE_FUNCTION_NAMES) { + assertTrue("Multimap should contain: " + name, multimap.containsKey(name)); + Collection<BuiltInFunctionInfo> infos = multimap.get(name); + assertEquals("Expected exactly 1 registered function for " + name, 1, infos.size()); + BuiltInFunctionInfo info = infos.iterator().next(); + assertEquals(name, info.getName()); + } + + // A name that is not registered must not resolve + assertFalse(multimap.containsKey("NON_EXISTENT_DISTANCE_FUNC")); + } + + @Test + public void testParseNodeFactoryDirectMethods() { + ParseNodeFactory factory = new ParseNodeFactory(); + ParseNode c1 = new ColumnParseNode(null, "V1", null); + ParseNode c2 = new ColumnParseNode(null, "V2", null); + List<ParseNode> children = Arrays.asList(c1, c2); + + assertDistanceNode(L2DistanceFunction.NAME, factory.l2Distance(children)); + assertDistanceNode(CosineDistanceFunction.NAME, factory.cosineDistance(children)); + assertDistanceNode(InnerProductDistanceFunction.NAME, factory.innerProductDistance(children)); + for (String name : DISTANCE_FUNCTION_NAMES) { + assertDistanceNode(name, factory.function(name, children)); + } + } + + @Test + public void testSQLParsingDistanceFunctions() throws Exception { + for (String name : DISTANCE_FUNCTION_NAMES) { + for (String spelling : new String[] { name, name.toLowerCase() }) { + SelectStatement select = + (SelectStatement) new SQLParser("SELECT " + spelling + "(v1, v2) FROM t") + .parseStatement(); + ParseNode node = select.getSelect().get(0).getNode(); + assertDistanceNode(name, node); + assertEquals(2, node.getChildren().size()); + } + } + } + + @Test + public void testEndToEndSQLCompilation() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute( + "CREATE TABLE t_vector_func_test (" + "pk INTEGER PRIMARY KEY, " + "v1 VECTOR(FLOAT, 3), " + + "v2 VECTOR(FLOAT, 3), " + "vd1 VECTOR(DOUBLE, 4), " + "vd2 VECTOR(DOUBLE, 4))"); + + String[] queries = { "SELECT L2_DISTANCE(v1, v2) FROM t_vector_func_test", + "SELECT L2_DISTANCE_SQUARED(v1, v2) FROM t_vector_func_test", + "SELECT COSINE_DISTANCE(v1, v2) FROM t_vector_func_test", + "SELECT INNER_PRODUCT(v1, v2) FROM t_vector_func_test", + "SELECT L2_DISTANCE(vd1, vd2) FROM t_vector_func_test", + "SELECT L2_DISTANCE_SQUARED(vd1, vd2) FROM t_vector_func_test", + "SELECT COSINE_DISTANCE(vd1, vd2) FROM t_vector_func_test", + "SELECT INNER_PRODUCT(vd1, vd2) FROM t_vector_func_test", + "SELECT pk FROM t_vector_func_test WHERE L2_DISTANCE(v1, v2) < 2.5", + "SELECT pk FROM t_vector_func_test ORDER BY COSINE_DISTANCE(v1, v2) LIMIT 10", + "SELECT L2_DISTANCE(v1, v2) + 1.0 AS dist FROM t_vector_func_test" }; + + for (String sql : queries) { + try (PreparedStatement stmt = conn.prepareStatement(sql)) { + PhoenixPreparedStatement pStmt = stmt.unwrap(PhoenixPreparedStatement.class); + QueryPlan plan = pStmt.compileQuery(); + assertNotNull("Plan should not be null for query: " + sql, plan); + } + } + } + } + + @Test + public void testCompilationFailsOnDimensionMismatch() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute("CREATE TABLE t_dim_mismatch (" + "pk INTEGER PRIMARY KEY, " + + "v3 VECTOR(FLOAT, 3), " + "v4 VECTOR(FLOAT, 4))"); + + try (PreparedStatement stmt = + conn.prepareStatement("SELECT L2_DISTANCE(v3, v4) FROM t_dim_mismatch")) { + PhoenixPreparedStatement pStmt = stmt.unwrap(PhoenixPreparedStatement.class); + pStmt.compileQuery(); + fail("Expected compilation to fail due to dimension mismatch"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.VECTOR_DIMENSION_MISMATCH.getErrorCode(), e.getErrorCode()); + } + } + } + + @Test + public void testCompilationFailsOnNonVectorArgument() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute( + "CREATE TABLE t_non_vector (" + "pk INTEGER PRIMARY KEY, " + "v VECTOR(FLOAT, 3))"); + + try (PreparedStatement stmt = + conn.prepareStatement("SELECT L2_DISTANCE(pk, v) FROM t_non_vector")) { + PhoenixPreparedStatement pStmt = stmt.unwrap(PhoenixPreparedStatement.class); + pStmt.compileQuery(); + fail("Expected compilation to fail due to non-vector argument"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.TYPE_MISMATCH.getErrorCode(), e.getErrorCode()); + } + } + } + + @Test + public void testCompilationFailsOnUnknownFunction() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute("CREATE TABLE t_unknown_func (" + "pk INTEGER PRIMARY KEY, " + + "v1 VECTOR(FLOAT, 3), " + "v2 VECTOR(FLOAT, 3))"); + + try (PreparedStatement stmt = + conn.prepareStatement("SELECT UNKNOWN_DISTANCE_FUNC(v1, v2) FROM t_unknown_func")) { + PhoenixPreparedStatement pStmt = stmt.unwrap(PhoenixPreparedStatement.class); + pStmt.compileQuery(); + fail("Expected compilation to fail for unknown function"); + } catch (SQLException e) { + // Compilation fails because the function name does not resolve + assertNotNull(e.getMessage()); + } + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/function/VectorDistanceUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/VectorDistanceUtilTest.java new file mode 100644 index 0000000000..4ca4732ace --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/VectorDistanceUtilTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.expression.BaseTerminalExpression; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDouble; +import org.apache.phoenix.schema.types.PVectorDouble; +import org.apache.phoenix.schema.types.PVectorFloat; +import org.junit.Test; + +public class VectorDistanceUtilTest { + + private static final double DELTA = 1e-5; + + /** + * Checks that each bounded method returns Double.MAX_VALUE above the bound and the exact distance + * at or below it. + */ + @Test + public void testEarlyTerminationCorrectness() { + byte[] b1 = PVectorFloat.INSTANCE.toBytes(new float[] { 3.0f, 0.0f }); + byte[] b2 = PVectorFloat.INSTANCE.toBytes(new float[] { 0.0f, 4.0f }); + + assertEquals(Double.MAX_VALUE, VectorDistanceUtil.l2DistanceWithBound(b1, 0, b2, 0, 2, 1.0), + 0.0); + assertEquals(5.0, VectorDistanceUtil.l2DistanceWithBound(b1, 0, b2, 0, 2, 10.0), DELTA); + + byte[] sqB1 = PVectorFloat.INSTANCE.toBytes(new float[] { 1.0f, 2.0f }); + byte[] sqB2 = PVectorFloat.INSTANCE.toBytes(new float[] { 0.0f, 0.0f }); + assertEquals(Double.MAX_VALUE, + VectorDistanceUtil.l2DistanceSquaredWithBound(sqB1, 0, sqB2, 0, 2, 1.0), 0.0); + assertEquals(5.0, VectorDistanceUtil.l2DistanceSquaredWithBound(sqB1, 0, sqB2, 0, 2, 10.0), + DELTA); + + assertEquals(Double.MAX_VALUE, + VectorDistanceUtil.l2DistanceSquaredWithBound(b1, 0, b2, 0, 2, 1.0), 0.0); + assertEquals(25.0, VectorDistanceUtil.l2DistanceSquaredWithBound(b1, 0, b2, 0, 2, 30.0), DELTA); + + // Inner product and cosine compute the full distance and then apply the bound + assertEquals(Double.MAX_VALUE, + VectorDistanceUtil.innerProductDistanceWithBound(b1, 0, b2, 0, 2, -1.0), 0.0); + assertEquals(0.0, + VectorDistanceUtil.innerProductDistanceWithBound(b1, 0, b2, 0, 2, Double.MAX_VALUE), DELTA); + assertEquals(Double.MAX_VALUE, VectorDistanceUtil.cosineDistanceWithBound(b1, 0, b2, 0, 2, 0.5), + 0.0); + assertEquals(1.0, VectorDistanceUtil.cosineDistanceWithBound(b1, 0, b2, 0, 2, 1.5), DELTA); + } + + /** + * Checks that the bounded scalar L2 kernels stop early. The operands are shorter than the given + * dimension, so a kernel that does not stop early fails with an exception. + */ + @Test + public void testEarlyTerminationStopsReading() { + int dim = 128; + int readable = dim / 4; + float[] v1 = new float[readable]; + Arrays.fill(v1, (float) (5.0 / Math.sqrt(dim))); + float[] v2 = new float[readable]; + // The partial sum passes the bound before the end of the short operands + assertEquals(Double.MAX_VALUE, ScalarDistanceKernel.l2DistanceSquared(v1, v2, dim, 1.0), 0.0); + byte[] b1 = PVectorFloat.INSTANCE.toBytes(v1); + byte[] b2 = PVectorFloat.INSTANCE.toBytes(v2); + assertEquals(Double.MAX_VALUE, ScalarDistanceKernel.l2DistanceSquared(b1, 0, b2, 0, dim, 1.0), + 0.0); + } + + /** + * Checks that the scalar reference methods and the query time methods give the same distances. + */ + @Test + public void testScalarAndPackedAgree() { + Random rng = new Random(3); + for (int dim : new int[] { 1, 5, 64, 300 }) { + float[] a = new float[dim]; + float[] b = new float[dim]; + for (int i = 0; i < dim; i++) { + a[i] = (rng.nextFloat() - 0.5f) * 20.0f; + b[i] = (rng.nextFloat() - 0.5f) * 20.0f; + } + byte[] pa = PVectorFloat.INSTANCE.toBytes(a); + byte[] pb = PVectorFloat.INSTANCE.toBytes(b); + assertClose(VectorDistanceUtil.scalarL2DistanceSquared(a, b, dim), + VectorDistanceUtil.l2DistanceSquaredWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + assertClose(VectorDistanceUtil.scalarInnerProductDistance(a, b, dim), + VectorDistanceUtil.innerProductDistanceWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + assertClose(VectorDistanceUtil.scalarCosineDistance(a, b, dim), + VectorDistanceUtil.cosineDistanceWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + } + } + + private static void assertClose(double expected, double actual) { + assertEquals(expected, actual, DELTA * Math.max(1.0, Math.abs(expected))); + } + + /** + * Checks that a distance function applies the upper bound that a top-N scan sets on it. + */ + @Test + public void testDistanceFunctionEvaluateWithBound() throws Exception { + Expression child1 = + LiteralExpression.newConstant(new float[] { 3.0f, 0.0f }, PVectorFloat.INSTANCE, 2, null); + Expression child2 = + LiteralExpression.newConstant(new float[] { 0.0f, 4.0f }, PVectorFloat.INSTANCE, 2, null); + L2DistanceFunction func = new L2DistanceFunction(Arrays.asList(child1, child2)); + + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + func.setDistanceUpperBound(10.0); + assertTrue(func.evaluate(null, ptr)); + assertEquals(5.0, ((Number) PDouble.INSTANCE.toObject(ptr)).doubleValue(), DELTA); + + func.setDistanceUpperBound(1.0); + assertTrue(func.evaluate(null, ptr)); + assertEquals(Double.MAX_VALUE, ((Number) PDouble.INSTANCE.toObject(ptr)).doubleValue(), 0.0); + } + + /** + * A vector operand is a cell value at a random offset inside a larger buffer. Each distance + * function must read only the operand bytes, for float, double, and mixed precision operands in + * each sort order. The result must agree with a double precision reference from this test. + */ + @Test + public void testDistanceFunctionsReadOperandsAtOffsets() { + Random rng = new Random(17); + SortOrder[] orders = { SortOrder.ASC, SortOrder.DESC }; + for (int dim : new int[] { 1, 7, 64, 129 }) { + for (int trial = 0; trial < 12; trial++) { + float[] a = new float[dim]; + float[] b = new float[dim]; + for (int i = 0; i < dim; i++) { + a[i] = (rng.nextFloat() - 0.5f) * 20.0f; + b[i] = (rng.nextFloat() - 0.5f) * 20.0f; + } + boolean aIsDouble = trial % 3 == 1; + boolean bIsDouble = trial % 3 != 0; + SortOrder aOrder = orders[(trial / 3) % 2]; + SortOrder bOrder = orders[(trial / 6) % 2]; + List<Expression> args = + Arrays.asList(new OffsetVectorExpression(rng, a, aIsDouble, aOrder, 1 + trial % 5), + new OffsetVectorExpression(rng, b, bIsDouble, bOrder, 3 + trial % 7)); + + double sumSq = 0.0; + double dot = 0.0; + double normA = 0.0; + double normB = 0.0; + double terms = 0.0; + for (int i = 0; i < dim; i++) { + double diff = (double) a[i] - b[i]; + sumSq += diff * diff; + dot += (double) a[i] * b[i]; + normA += (double) a[i] * a[i]; + normB += (double) b[i] * b[i]; + terms += Math.abs((double) a[i] * b[i]); + } + String ctx = "dim=" + dim + " trial=" + trial; + assertNear(ctx, sumSq, evaluate(new L2DistanceSquaredFunction(args)), sumSq); + assertNear(ctx, Math.sqrt(sumSq), evaluate(new L2DistanceFunction(args)), Math.sqrt(sumSq)); + assertNear(ctx, -dot, evaluate(new InnerProductDistanceFunction(args)), terms); + assertNear(ctx, 1.0 - dot / (Math.sqrt(normA) * Math.sqrt(normB)), + evaluate(new CosineDistanceFunction(args)), terms / Math.sqrt(normA * normB)); + } + } + } + + private static double evaluate(DistanceFunction function) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + assertTrue(function.evaluate(null, ptr)); + return ((Number) PDouble.INSTANCE.toObject(ptr)).doubleValue(); + } + + private static void assertNear(String ctx, double expected, double actual, double magnitude) { + assertEquals(ctx, expected, actual, 1e-4 * Math.max(1.0, magnitude)); + } + + /** + * A vector operand encoded between random bytes. A distance function must not read those bytes. + */ + private static class OffsetVectorExpression extends BaseTerminalExpression { + private final byte[] buf; + private final int offset; + private final int length; + private final boolean isDouble; + private final SortOrder sortOrder; + private final int dim; + + OffsetVectorExpression(Random rng, float[] v, boolean isDouble, SortOrder sortOrder, + int offset) { + this.isDouble = isDouble; + this.sortOrder = sortOrder; + this.offset = offset; + this.dim = v.length; + this.length = v.length * (isDouble ? Bytes.SIZEOF_DOUBLE : Bytes.SIZEOF_FLOAT); + this.buf = new byte[offset + length + 16]; + rng.nextBytes(buf); + byte[] encoded = isDouble + ? PVectorDouble.INSTANCE.toBytes(toDoubles(v), sortOrder) + : PVectorFloat.INSTANCE.toBytes(v, sortOrder); + System.arraycopy(encoded, 0, buf, offset, length); + } + + private static double[] toDoubles(float[] v) { + double[] d = new double[v.length]; + for (int i = 0; i < v.length; i++) { + d[i] = v[i]; + } + return d; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + ptr.set(buf, offset, length); + return true; + } + + @Override + public PDataType getDataType() { + return isDouble ? PVectorDouble.INSTANCE : PVectorFloat.INSTANCE; + } + + @Override + public Integer getMaxLength() { + return dim; + } + + @Override + public SortOrder getSortOrder() { + return sortOrder; + } + + @Override + public <T> T accept(ExpressionVisitor<T> visitor) { + return null; + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/parse/DistanceOperatorParseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/parse/DistanceOperatorParseTest.java new file mode 100644 index 0000000000..eec9cd4734 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/parse/DistanceOperatorParseTest.java @@ -0,0 +1,539 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.phoenix.util.QueryUtil; +import org.junit.Test; + +/** + * Parser tests for the infix vector distance operators (<->, <=>, <#>). Each operator must parse to + * the same parse node as its distance function. The tests also examine the precedence and + * associativity of the operators relative to arithmetic, comparison, and logical operators. + */ +public class DistanceOperatorParseTest { + + private static boolean isDistance(ParseNode node, String name) { + return node instanceof DistanceFunctionParseNode + && name.equals(((FunctionParseNode) node).getName()); + } + + @Test + public void testParseL2DistanceOperator() throws Exception { + String sql = "SELECT v1 <-> v2 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + assertEquals(1, select.getSelect().size()); + ParseNode node = select.getSelect().get(0).getNode(); + assertTrue("Expected L2_DISTANCE node, got: " + node.getClass().getSimpleName(), + isDistance(node, "L2_DISTANCE")); + assertEquals(2, node.getChildren().size()); + assertTrue(node.getChildren().get(0) instanceof ColumnParseNode); + assertTrue(node.getChildren().get(1) instanceof ColumnParseNode); + assertEquals("V1", ((ColumnParseNode) node.getChildren().get(0)).getName()); + assertEquals("V2", ((ColumnParseNode) node.getChildren().get(1)).getName()); + } + + @Test + public void testParseCosineDistanceOperator() throws Exception { + String sql = "SELECT v1 <=> v2 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + assertEquals(1, select.getSelect().size()); + ParseNode node = select.getSelect().get(0).getNode(); + assertTrue("Expected COSINE_DISTANCE node, got: " + node.getClass().getSimpleName(), + isDistance(node, "COSINE_DISTANCE")); + assertEquals(2, node.getChildren().size()); + assertTrue(node.getChildren().get(0) instanceof ColumnParseNode); + assertTrue(node.getChildren().get(1) instanceof ColumnParseNode); + assertEquals("V1", ((ColumnParseNode) node.getChildren().get(0)).getName()); + assertEquals("V2", ((ColumnParseNode) node.getChildren().get(1)).getName()); + } + + @Test + public void testParseInnerProductDistanceOperator() throws Exception { + String sql = "SELECT v1 <#> v2 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + assertEquals(1, select.getSelect().size()); + ParseNode node = select.getSelect().get(0).getNode(); + assertTrue("Expected INNER_PRODUCT node, got: " + node.getClass().getSimpleName(), + isDistance(node, "INNER_PRODUCT")); + assertEquals(2, node.getChildren().size()); + assertTrue(node.getChildren().get(0) instanceof ColumnParseNode); + assertTrue(node.getChildren().get(1) instanceof ColumnParseNode); + assertEquals("V1", ((ColumnParseNode) node.getChildren().get(0)).getName()); + assertEquals("V2", ((ColumnParseNode) node.getChildren().get(1)).getName()); + } + + @Test + public void testOperatorFunctionEquivalence() throws Exception { + // The <-> operator gives the same parse node as L2_DISTANCE. + SelectStatement opL2 = + (SelectStatement) new SQLParser("SELECT v1 <-> v2 FROM t").parseStatement(); + SelectStatement fnL2 = + (SelectStatement) new SQLParser("SELECT L2_DISTANCE(v1, v2) FROM t").parseStatement(); + ParseNode opL2Node = opL2.getSelect().get(0).getNode(); + ParseNode fnL2Node = fnL2.getSelect().get(0).getNode(); + assertTrue(isDistance(opL2Node, "L2_DISTANCE")); + assertEquals(fnL2Node.getClass(), opL2Node.getClass()); + assertEquals(fnL2Node, opL2Node); + + // The <=> operator gives the same parse node as COSINE_DISTANCE. + SelectStatement opCos = + (SelectStatement) new SQLParser("SELECT v1 <=> v2 FROM t").parseStatement(); + SelectStatement fnCos = + (SelectStatement) new SQLParser("SELECT COSINE_DISTANCE(v1, v2) FROM t").parseStatement(); + ParseNode opCosNode = opCos.getSelect().get(0).getNode(); + ParseNode fnCosNode = fnCos.getSelect().get(0).getNode(); + assertTrue(isDistance(opCosNode, "COSINE_DISTANCE")); + assertEquals(fnCosNode.getClass(), opCosNode.getClass()); + assertEquals(fnCosNode, opCosNode); + + // The <#> operator gives the same parse node as INNER_PRODUCT. + SelectStatement opIp = + (SelectStatement) new SQLParser("SELECT v1 <#> v2 FROM t").parseStatement(); + SelectStatement fnIp = + (SelectStatement) new SQLParser("SELECT INNER_PRODUCT(v1, v2) FROM t").parseStatement(); + ParseNode opIpNode = opIp.getSelect().get(0).getNode(); + ParseNode fnIpNode = fnIp.getSelect().get(0).getNode(); + assertTrue(isDistance(opIpNode, "INNER_PRODUCT")); + assertEquals(fnIpNode.getClass(), opIpNode.getClass()); + assertEquals(fnIpNode, opIpNode); + } + + @Test + public void testPrecedenceDistanceVsComparison() throws Exception { + // Distance operators have a higher precedence than comparison operators. + String sql = "SELECT v1 <-> v2 < 0.5 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode root = select.getSelect().get(0).getNode(); + assertTrue("Expected LessThanParseNode, got: " + root.getClass().getSimpleName(), + root instanceof LessThanParseNode); + assertEquals(2, root.getChildren().size()); + ParseNode lhs = root.getChildren().get(0); + ParseNode rhs = root.getChildren().get(1); + assertTrue("LHS should be L2_DISTANCE node, got: " + lhs.getClass().getSimpleName(), + isDistance(lhs, "L2_DISTANCE")); + assertTrue("RHS should be LiteralParseNode, got: " + rhs.getClass().getSimpleName(), + rhs instanceof LiteralParseNode); + assertEquals(2, lhs.getChildren().size()); + assertEquals("V1", ((ColumnParseNode) lhs.getChildren().get(0)).getName()); + assertEquals("V2", ((ColumnParseNode) lhs.getChildren().get(1)).getName()); + } + + @Test + public void testPrecedenceArithmeticVsDistance() throws Exception { + // Additive and multiplicative operators have a higher precedence than distance operators. + String sql = "SELECT v1 + v2 <-> v3 * v4 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode root = select.getSelect().get(0).getNode(); + assertTrue("Expected L2_DISTANCE node, got: " + root.getClass().getSimpleName(), + isDistance(root, "L2_DISTANCE")); + assertEquals(2, root.getChildren().size()); + ParseNode lhs = root.getChildren().get(0); + ParseNode rhs = root.getChildren().get(1); + assertTrue("LHS should be AddParseNode, got: " + lhs.getClass().getSimpleName(), + lhs instanceof AddParseNode); + assertTrue("RHS should be MultiplyParseNode, got: " + rhs.getClass().getSimpleName(), + rhs instanceof MultiplyParseNode); + } + + @Test + public void testParenthesesOverridePrecedence() throws Exception { + // Parentheses change the default precedence. + String sql = "SELECT (v1 <-> v2) + 1.0 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode root = select.getSelect().get(0).getNode(); + assertTrue("Expected AddParseNode, got: " + root.getClass().getSimpleName(), + root instanceof AddParseNode); + assertEquals(2, root.getChildren().size()); + ParseNode lhs = root.getChildren().get(0); + assertTrue("LHS should be L2_DISTANCE node, got: " + lhs.getClass().getSimpleName(), + isDistance(lhs, "L2_DISTANCE")); + } + + @Test + public void testWhereClauseWithDistanceOperator() throws Exception { + String sql = "SELECT * FROM t WHERE v1 <-> v2 < 0.5"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode where = select.getWhere(); + assertNotNull("WHERE clause should not be null", where); + assertTrue("WHERE should be LessThanParseNode", where instanceof LessThanParseNode); + ParseNode lhs = where.getChildren().get(0); + assertTrue("LHS of WHERE comparison should be L2_DISTANCE node", + isDistance(lhs, "L2_DISTANCE")); + } + + @Test + public void testOrderByWithDistanceOperator() throws Exception { + String sql = "SELECT doc_id FROM documents ORDER BY embedding <=> :1 LIMIT 10"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + List<OrderByNode> orderBy = select.getOrderBy(); + assertNotNull(orderBy); + assertEquals(1, orderBy.size()); + ParseNode orderNode = orderBy.get(0).getNode(); + assertTrue("ORDER BY node should be COSINE_DISTANCE node", + isDistance(orderNode, "COSINE_DISTANCE")); + assertEquals(2, orderNode.getChildren().size()); + assertTrue(orderNode.getChildren().get(0) instanceof ColumnParseNode); + assertTrue(orderNode.getChildren().get(1) instanceof BindParseNode); + } + + @Test + public void testNoSpaceDistanceOperator() throws Exception { + String sql = "SELECT v1<->v2, v1<=>v2, v1<#>v2 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + assertEquals(3, select.getSelect().size()); + assertTrue(isDistance(select.getSelect().get(0).getNode(), "L2_DISTANCE")); + assertTrue(isDistance(select.getSelect().get(1).getNode(), "COSINE_DISTANCE")); + assertTrue(isDistance(select.getSelect().get(2).getNode(), "INNER_PRODUCT")); + } + + @Test + public void testComplexLogicalPrecedence() throws Exception { + // The precedence order, from highest to lowest, is: distance, comparison, NOT, AND, OR. + String sql = + "SELECT * FROM t WHERE v1 <-> v2 < 0.5 AND NOT v3 <=> v4 >= 0.8 OR v1 <#> v2 < -10.0"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode where = select.getWhere(); + assertNotNull(where); + assertTrue("Root WHERE should be OrParseNode, got: " + where.getClass().getSimpleName(), + where instanceof OrParseNode); + assertEquals(2, where.getChildren().size()); + + ParseNode andNode = where.getChildren().get(0); + assertTrue("Left of OR should be AndParseNode", andNode instanceof AndParseNode); + assertEquals(2, andNode.getChildren().size()); + + ParseNode leftComp = andNode.getChildren().get(0); + assertTrue("Left of AND should be LessThanParseNode", leftComp instanceof LessThanParseNode); + assertTrue("LHS of < should be L2_DISTANCE node", + isDistance(leftComp.getChildren().get(0), "L2_DISTANCE")); + assertTrue("RHS of < should be LiteralParseNode", + leftComp.getChildren().get(1) instanceof LiteralParseNode); + + ParseNode notNode = andNode.getChildren().get(1); + assertTrue("Right of AND should be NotParseNode", notNode instanceof NotParseNode); + ParseNode notChild = notNode.getChildren().get(0); + assertTrue("Child of NOT should be GreaterThanOrEqualParseNode", + notChild instanceof GreaterThanOrEqualParseNode); + assertTrue("LHS of >= should be COSINE_DISTANCE node", + isDistance(notChild.getChildren().get(0), "COSINE_DISTANCE")); + assertTrue("RHS of >= should be LiteralParseNode", + notChild.getChildren().get(1) instanceof LiteralParseNode); + + ParseNode rightComp = where.getChildren().get(1); + assertTrue("Right of OR should be LessThanParseNode", rightComp instanceof LessThanParseNode); + assertTrue("LHS of < should be INNER_PRODUCT node", + isDistance(rightComp.getChildren().get(0), "INNER_PRODUCT")); + } + + @Test + public void testFullArithmeticSpectrumWithDistance() throws Exception { + // The precedence order, from highest to lowest, is: multiplicative, additive, distance, + // comparison. + String sql = + "SELECT (v1 + v2 * 2.0) <-> (v3 - v4 / 2.0 % 1.0) < (threshold + 0.1) * 2.0 FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode root = select.getSelect().get(0).getNode(); + assertTrue("Expected LessThanParseNode, got: " + root.getClass().getSimpleName(), + root instanceof LessThanParseNode); + + ParseNode distNode = root.getChildren().get(0); + assertTrue("LHS of < should be L2_DISTANCE node", isDistance(distNode, "L2_DISTANCE")); + assertEquals(2, distNode.getChildren().size()); + + ParseNode op1 = distNode.getChildren().get(0); + assertTrue("Operand 1 of <-> should be AddParseNode", op1 instanceof AddParseNode); + assertTrue("Add child 0 should be ColumnParseNode", + op1.getChildren().get(0) instanceof ColumnParseNode); + assertTrue("Add child 1 should be MultiplyParseNode", + op1.getChildren().get(1) instanceof MultiplyParseNode); + + ParseNode op2 = distNode.getChildren().get(1); + assertTrue("Operand 2 of <-> should be SubtractParseNode", op2 instanceof SubtractParseNode); + assertTrue("Subtract child 0 should be ColumnParseNode", + op2.getChildren().get(0) instanceof ColumnParseNode); + ParseNode modNode = op2.getChildren().get(1); + assertTrue("Subtract child 1 should be ModulusParseNode", modNode instanceof ModulusParseNode); + assertTrue("Modulus child 0 should be DivideParseNode", + modNode.getChildren().get(0) instanceof DivideParseNode); + + ParseNode rhs = root.getChildren().get(1); + assertTrue("RHS of < should be MultiplyParseNode", rhs instanceof MultiplyParseNode); + assertTrue("Multiply child 0 should be AddParseNode", + rhs.getChildren().get(0) instanceof AddParseNode); + } + + @Test + public void testBetweenClauseWithDistanceExpressions() throws Exception { + // A distance expression can be the operand of BETWEEN. + String sql1 = "SELECT * FROM t WHERE v1 <-> v2 BETWEEN 0.1 AND 0.9"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode where1 = select1.getWhere(); + assertTrue("WHERE should be BetweenParseNode", where1 instanceof BetweenParseNode); + assertEquals(3, where1.getChildren().size()); + assertTrue("BETWEEN target should be L2_DISTANCE node", + isDistance(where1.getChildren().get(0), "L2_DISTANCE")); + assertTrue("BETWEEN lower bound should be LiteralParseNode", + where1.getChildren().get(1) instanceof LiteralParseNode); + assertTrue("BETWEEN upper bound should be LiteralParseNode", + where1.getChildren().get(2) instanceof LiteralParseNode); + + // Distance expressions can be the range limits of BETWEEN. + String sql2 = "SELECT * FROM t WHERE target_dist BETWEEN v1 <-> v2 AND v3 <=> v4"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + ParseNode where2 = select2.getWhere(); + assertTrue("WHERE should be BetweenParseNode", where2 instanceof BetweenParseNode); + assertEquals(3, where2.getChildren().size()); + assertTrue("BETWEEN target should be ColumnParseNode", + where2.getChildren().get(0) instanceof ColumnParseNode); + assertTrue("BETWEEN lower bound should be L2_DISTANCE node", + isDistance(where2.getChildren().get(1), "L2_DISTANCE")); + assertTrue("BETWEEN upper bound should be COSINE_DISTANCE node", + isDistance(where2.getChildren().get(2), "COSINE_DISTANCE")); + } + + @Test + public void testInListWithDistanceExpression() throws Exception { + // A distance expression can be the operand of IN. + String sql = "SELECT * FROM t WHERE v1 <-> v2 IN (0.1, 0.2, 0.3)"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode where = select.getWhere(); + assertTrue("WHERE should be InListParseNode", where instanceof InListParseNode); + List<ParseNode> children = where.getChildren(); + assertEquals(4, children.size()); + assertTrue("IN target should be L2_DISTANCE node", isDistance(children.get(0), "L2_DISTANCE")); + assertTrue("IN element 1 should be LiteralParseNode", + children.get(1) instanceof LiteralParseNode); + assertTrue("IN element 2 should be LiteralParseNode", + children.get(2) instanceof LiteralParseNode); + assertTrue("IN element 3 should be LiteralParseNode", + children.get(3) instanceof LiteralParseNode); + } + + @Test + public void testCaseWhenWithDistanceExpressions() throws Exception { + // Distance expressions can be in the WHEN conditions and in the results of CASE. + String sql = "SELECT CASE " + "WHEN v1 <=> v2 < 0.2 THEN v1 <-> v2 " + + "WHEN v1 <=> v2 < 0.5 THEN (v1 <-> v2) * 1.5 " + "ELSE v1 <#> v2 END FROM t"; + SQLParser parser = new SQLParser(sql); + SelectStatement select = (SelectStatement) parser.parseStatement(); + ParseNode root = select.getSelect().get(0).getNode(); + assertTrue("Expected CaseParseNode", root instanceof CaseParseNode); + List<ParseNode> children = root.getChildren(); + assertEquals(5, children.size()); + + assertTrue("THEN 1 should be L2_DISTANCE node", isDistance(children.get(0), "L2_DISTANCE")); + assertTrue("WHEN 1 should be LessThanParseNode", children.get(1) instanceof LessThanParseNode); + assertTrue("WHEN 1 LHS should be COSINE_DISTANCE node", + isDistance(children.get(1).getChildren().get(0), "COSINE_DISTANCE")); + + assertTrue("THEN 2 should be MultiplyParseNode", children.get(2) instanceof MultiplyParseNode); + assertTrue("THEN 2 child 0 should be L2_DISTANCE node", + isDistance(children.get(2).getChildren().get(0), "L2_DISTANCE")); + assertTrue("WHEN 2 should be LessThanParseNode", children.get(3) instanceof LessThanParseNode); + assertTrue("WHEN 2 LHS should be COSINE_DISTANCE node", + isDistance(children.get(3).getChildren().get(0), "COSINE_DISTANCE")); + + assertTrue("ELSE should be INNER_PRODUCT node", isDistance(children.get(4), "INNER_PRODUCT")); + } + + @Test + public void testChainedDistanceOperatorsAssociativity() throws Exception { + // A chain of the same distance operator associates from left to right. + String sql1 = "SELECT v1 <-> v2 <-> v3 FROM t"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode root1 = select1.getSelect().get(0).getNode(); + assertTrue("Root should be L2_DISTANCE node", isDistance(root1, "L2_DISTANCE")); + assertEquals(2, root1.getChildren().size()); + assertTrue("LHS should be L2_DISTANCE node", + isDistance(root1.getChildren().get(0), "L2_DISTANCE")); + assertTrue("RHS should be ColumnParseNode", + root1.getChildren().get(1) instanceof ColumnParseNode); + assertEquals("V3", ((ColumnParseNode) root1.getChildren().get(1)).getName()); + + // A chain of different distance operators also associates from left to right. + String sql2 = "SELECT v1 <-> v2 <=> v3 <#> v4 FROM t"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + ParseNode root2 = select2.getSelect().get(0).getNode(); + assertTrue("Root should be INNER_PRODUCT node", isDistance(root2, "INNER_PRODUCT")); + assertEquals(2, root2.getChildren().size()); + + ParseNode cosNode = root2.getChildren().get(0); + assertTrue("LHS of <#> should be COSINE_DISTANCE node", isDistance(cosNode, "COSINE_DISTANCE")); + assertTrue("LHS of <=> should be L2_DISTANCE node", + isDistance(cosNode.getChildren().get(0), "L2_DISTANCE")); + assertEquals("V3", ((ColumnParseNode) cosNode.getChildren().get(1)).getName()); + assertEquals("V4", ((ColumnParseNode) root2.getChildren().get(1)).getName()); + } + + @Test + public void testUnaryMinusPrecedenceWithDistance() throws Exception { + // Unary minus on a vector operand has a higher precedence than a distance operator. + String sql1 = "SELECT -v1 <-> v2 FROM t"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode root1 = select1.getSelect().get(0).getNode(); + assertTrue("Root should be L2_DISTANCE node", isDistance(root1, "L2_DISTANCE")); + ParseNode lhs1 = root1.getChildren().get(0); + assertTrue("Operand with unary minus should be MultiplyParseNode (negate representation)", + lhs1 instanceof MultiplyParseNode); + assertTrue(root1.getChildren().get(1) instanceof ColumnParseNode); + + // With parentheses, unary minus applies to the result of the distance expression. + String sql2 = "SELECT -(v1 <-> v2) FROM t"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + ParseNode root2 = select2.getSelect().get(0).getNode(); + assertTrue("Root should be MultiplyParseNode (negate representation)", + root2 instanceof MultiplyParseNode); + assertTrue("Child of negate should be L2_DISTANCE node", + isDistance(root2.getChildren().get(0), "L2_DISTANCE")); + } + + @Test + public void testNestedFunctionsAndComplexOrderBy() throws Exception { + // A distance expression can be an argument of a scalar function. + String sql1 = "SELECT ROUND(v1 <-> v2, 2) > ROUND(v3 <=> v4, 2) FROM t"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode root1 = select1.getSelect().get(0).getNode(); + assertTrue("Expected GreaterThanParseNode", root1 instanceof GreaterThanParseNode); + ParseNode leftFn = root1.getChildren().get(0); + ParseNode rightFn = root1.getChildren().get(1); + assertTrue("LHS should be FunctionParseNode", leftFn instanceof FunctionParseNode); + assertTrue("RHS should be FunctionParseNode", rightFn instanceof FunctionParseNode); + assertTrue("LHS function arg should be L2_DISTANCE node", + isDistance(leftFn.getChildren().get(0), "L2_DISTANCE")); + assertTrue("RHS function arg should be COSINE_DISTANCE node", + isDistance(rightFn.getChildren().get(0), "COSINE_DISTANCE")); + + // ORDER BY can use a weighted sum of two distance metrics. + String sql2 = "SELECT doc_id FROM documents " + + "ORDER BY (embedding <=> :1) * 0.7 + (embedding <-> :2) * 0.3 ASC, doc_id DESC"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + List<OrderByNode> orderBy = select2.getOrderBy(); + assertNotNull(orderBy); + assertEquals(2, orderBy.size()); + + OrderByNode order1 = orderBy.get(0); + assertTrue("First ORDER BY should be ascending", order1.isAscending()); + ParseNode orderNode1 = order1.getNode(); + assertTrue("Weighted combination should be AddParseNode", orderNode1 instanceof AddParseNode); + assertEquals(2, orderNode1.getChildren().size()); + + ParseNode term1 = orderNode1.getChildren().get(0); + assertTrue("Term 1 should be MultiplyParseNode", term1 instanceof MultiplyParseNode); + assertTrue("Term 1 child 0 should be COSINE_DISTANCE node", + isDistance(term1.getChildren().get(0), "COSINE_DISTANCE")); + + ParseNode term2 = orderNode1.getChildren().get(1); + assertTrue("Term 2 should be MultiplyParseNode", term2 instanceof MultiplyParseNode); + assertTrue("Term 2 child 0 should be L2_DISTANCE node", + isDistance(term2.getChildren().get(0), "L2_DISTANCE")); + + OrderByNode order2 = orderBy.get(1); + assertTrue("Second ORDER BY should be descending", !order2.isAscending()); + assertTrue("Second ORDER BY node should be ColumnParseNode", + order2.getNode() instanceof ColumnParseNode); + } + + @Test + public void testIsNullWithDistanceExpression() throws Exception { + // A distance expression can be the operand of IS NULL. + String sql1 = "SELECT * FROM t WHERE v1 <-> v2 IS NULL"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode where1 = select1.getWhere(); + assertTrue("WHERE should be IsNullParseNode", where1 instanceof IsNullParseNode); + assertTrue("Should be IS NULL (not negated)", !((IsNullParseNode) where1).isNegate()); + assertTrue("Child of IS NULL should be L2_DISTANCE node", + isDistance(where1.getChildren().get(0), "L2_DISTANCE")); + + // A distance expression can be the operand of IS NOT NULL. + String sql2 = "SELECT * FROM t WHERE v1 <=> v2 IS NOT NULL"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + ParseNode where2 = select2.getWhere(); + assertTrue("WHERE should be IsNullParseNode", where2 instanceof IsNullParseNode); + assertTrue("Should be IS NOT NULL (negated)", ((IsNullParseNode) where2).isNegate()); + assertTrue("Child of IS NOT NULL should be COSINE_DISTANCE node", + isDistance(where2.getChildren().get(0), "COSINE_DISTANCE")); + } + + @Test + public void testInterleavedArithmeticAndDistancePrecedence() throws Exception { + // Without parentheses, + binds before the distance operators, which associate from left to + // right. + String sql1 = "SELECT v1 <-> v2 + v3 <=> v4 FROM t"; + SelectStatement select1 = (SelectStatement) new SQLParser(sql1).parseStatement(); + ParseNode root1 = select1.getSelect().get(0).getNode(); + assertTrue("Root should be COSINE_DISTANCE node", isDistance(root1, "COSINE_DISTANCE")); + assertEquals(2, root1.getChildren().size()); + + ParseNode lhsCos = root1.getChildren().get(0); + assertTrue("LHS of <=> should be L2_DISTANCE node", isDistance(lhsCos, "L2_DISTANCE")); + ParseNode rhsCos = root1.getChildren().get(1); + assertTrue("RHS of <=> should be ColumnParseNode", rhsCos instanceof ColumnParseNode); + assertEquals("V4", ((ColumnParseNode) rhsCos).getName()); + + assertEquals("V1", ((ColumnParseNode) lhsCos.getChildren().get(0)).getName()); + ParseNode rhsL2 = lhsCos.getChildren().get(1); + assertTrue("RHS of <-> should be AddParseNode", rhsL2 instanceof AddParseNode); + assertEquals("V2", ((ColumnParseNode) rhsL2.getChildren().get(0)).getName()); + assertEquals("V3", ((ColumnParseNode) rhsL2.getChildren().get(1)).getName()); + + // Parentheses change the default precedence, so + applies to the two distance results. + String sql2 = "SELECT (v1 <-> v2) + (v3 <=> v4) FROM t"; + SelectStatement select2 = (SelectStatement) new SQLParser(sql2).parseStatement(); + ParseNode root2 = select2.getSelect().get(0).getNode(); + assertTrue("Root should be AddParseNode", root2 instanceof AddParseNode); + assertEquals(2, root2.getChildren().size()); + assertTrue("LHS of + should be L2_DISTANCE node", + isDistance(root2.getChildren().get(0), "L2_DISTANCE")); + assertTrue("RHS of + should be COSINE_DISTANCE node", + isDistance(root2.getChildren().get(1), "COSINE_DISTANCE")); + } + + /** + * Comparison operators that start with {@code <} must not parse as distance operators. For + * example, {@code a<-1} is a less-than comparison with a negative literal. + */ + @Test + public void testLessThanComparisonsUnaffected() throws Exception { + String[][] cases = + { { "a<-1", "<" }, { "a < -1", "<" }, { "a<=b", "<=" }, { "a<>b", "!=" }, { "a<b", "<" } }; + for (String[] c : cases) { + SelectStatement select = + (SelectStatement) new SQLParser("SELECT * FROM t WHERE " + c[0]).parseStatement(); + ParseNode where = select.getWhere(); + assertTrue(c[0] + " should parse as a comparison", where instanceof ComparisonParseNode); + assertEquals(c[0], c[1], QueryUtil.toSQL(((ComparisonParseNode) where).getFilterOp())); + } + } +} diff --git a/phoenix-core/src/test/java21/org/apache/phoenix/expression/function/PanamaDistanceKernelParityTest.java b/phoenix-core/src/test/java21/org/apache/phoenix/expression/function/PanamaDistanceKernelParityTest.java new file mode 100644 index 0000000000..70a22febf5 --- /dev/null +++ b/phoenix-core/src/test/java21/org/apache/phoenix/expression/function/PanamaDistanceKernelParityTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Random; +import org.apache.phoenix.schema.types.PVectorFloat; +import org.junit.Test; + +/** + * Parity tests that compare the Panama Vector API distance kernels with the scalar reference + * kernels. The tests cover many dimensions, unaligned value offsets, and the bound of the L2 + * kernel. Only the {@code java21} test profile runs them, because the Panama kernels need Java 21. + */ +public class PanamaDistanceKernelParityTest { + + private static final int[] DIMS = { 1, 3, 7, 8, 15, 16, 64, 100, 128, 129, 768, 1536 }; + private static final double TOLERANCE = 1e-5; + + private static float[] random(Random rng, int dim) { + float[] v = new float[dim]; + for (int i = 0; i < dim; i++) { + v[i] = (rng.nextFloat() - 0.5f) * 20.0f; + } + return v; + } + + private static void assertClose(String what, double expected, double actual) { + double scale = Math.max(1.0, Math.abs(expected)); + assertEquals(what, expected, actual, TOLERANCE * scale); + } + + @Test + public void testSimdKernelsAreActive() { + assertTrue("Panama kernels should be active under the java21 profile", + VectorDistanceUtil.isSimdEnabled()); + } + + @Test + public void testKernelParity() { + Random rng = new Random(42); + for (int dim : DIMS) { + for (int trial = 0; trial < 50; trial++) { + float[] a = random(rng, dim); + float[] b = random(rng, dim); + String ctx = "dim=" + dim + " trial=" + trial; + assertClose("l2sq " + ctx, + ScalarDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE), + PanamaDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE)); + assertClose("dot " + ctx, ScalarDistanceKernel.dotProduct(a, b, dim), + PanamaDistanceKernel.dotProduct(a, b, dim)); + assertClose("cosine " + ctx, ScalarDistanceKernel.cosineDistance(a, b, dim), + PanamaDistanceKernel.cosineDistance(a, b, dim)); + } + } + } + + @Test + public void testPackedParity() { + Random rng = new Random(7); + for (int dim : DIMS) { + float[] a = random(rng, dim); + float[] b = random(rng, dim); + byte[] pa = PVectorFloat.INSTANCE.toBytes(a); + byte[] pb = PVectorFloat.INSTANCE.toBytes(b); + String ctx = "dim=" + dim; + assertClose("l2sq " + ctx, ScalarDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE), + VectorDistanceUtil.l2DistanceSquaredWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + assertClose("l2 " + ctx, + Math.sqrt(ScalarDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE)), + VectorDistanceUtil.l2DistanceWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + assertClose("ip " + ctx, -ScalarDistanceKernel.dotProduct(a, b, dim), + VectorDistanceUtil.innerProductDistanceWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + assertClose("cosine " + ctx, ScalarDistanceKernel.cosineDistance(a, b, dim), + VectorDistanceUtil.cosineDistanceWithBound(pa, 0, pb, 0, dim, Double.MAX_VALUE)); + } + } + + @Test + public void testBoundedL2Semantics() { + Random rng = new Random(11); + for (int dim : DIMS) { + float[] a = random(rng, dim); + float[] b = random(rng, dim); + double exact = ScalarDistanceKernel.l2DistanceSquared(a, b, dim, Double.MAX_VALUE); + assertEquals(Double.MAX_VALUE, + PanamaDistanceKernel.l2DistanceSquared(a, b, dim, exact * 0.5), 0.0); + assertClose("dim=" + dim, exact, + PanamaDistanceKernel.l2DistanceSquared(a, b, dim, exact * 1.5)); + } + } +} diff --git a/phoenix-mapreduce-byo-shaded-hbase/pom.xml b/phoenix-mapreduce-byo-shaded-hbase/pom.xml index 477cd06345..283f13e8c0 100644 --- a/phoenix-mapreduce-byo-shaded-hbase/pom.xml +++ b/phoenix-mapreduce-byo-shaded-hbase/pom.xml @@ -602,4 +602,42 @@ </plugin> </plugins> </build> + + <profiles> + <profile> + <id>java21</id> + <!-- This profile marks the shaded jar as a multi-release jar. A Java 21 JVM then loads the + SIMD kernels of phoenix-core-client from META-INF/versions/21. --> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <filters combine.children="append"> + <!-- This filter keeps only the versioned classes of Phoenix. Shade does not + relocate the versioned classes of bundled libraries. If the jar keeps them, the + JVM loads them under their unrelocated names. --> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>%regex[META-INF/versions/[0-9]+/(?!org/apache/phoenix/).*]</exclude> + </excludes> + </filter> + </filters> + <transformers combine.children="append"> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <manifestEntries> + <Multi-Release>true</Multi-Release> + </manifestEntries> + </transformer> + </transformers> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> + </profiles> </project> diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml index 05e068f108..7176605848 100644 --- a/phoenix-server/pom.xml +++ b/phoenix-server/pom.xml @@ -569,4 +569,42 @@ </plugin> </plugins> </build> + + <profiles> + <profile> + <id>java21</id> + <!-- This profile marks the shaded jar as a multi-release jar. A Java 21 JVM then loads the + SIMD kernels of phoenix-core-client from META-INF/versions/21. --> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <filters combine.children="append"> + <!-- This filter keeps only the versioned classes of Phoenix. Shade does not + relocate the versioned classes of bundled libraries. If the jar keeps them, the + JVM loads them under their unrelocated names. --> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>%regex[META-INF/versions/[0-9]+/(?!org/apache/phoenix/).*]</exclude> + </excludes> + </filter> + </filters> + <transformers combine.children="append"> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <manifestEntries> + <Multi-Release>true</Multi-Release> + </manifestEntries> + </transformer> + </transformers> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> + </profiles> </project> diff --git a/pom.xml b/pom.xml index 58f24340a8..a544e19319 100644 --- a/pom.xml +++ b/pom.xml @@ -2509,6 +2509,14 @@ @{jacocoArgLine}</argLine> </properties> </profile> + <!-- The Java 21 profile adds the Panama Vector API incubator module to the test JVM. --> + <profile> + <id>java21</id> + <properties> + <phoenix-surefire.jdk21.flags>-Djava.security.manager=allow + --add-modules jdk.incubator.vector</phoenix-surefire.jdk21.flags> + </properties> + </profile> <profile> <id>build-with-jdk21</id> <activation>
