>From Hongyu Shi <[email protected]>: Hongyu Shi has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21653?usp=email )
Change subject: [ASTERIXDB-3817][COMP][RT] Add isVector() for embedding validation ...................................................................... [ASTERIXDB-3817][COMP][RT] Add isVector() for embedding validation - user model changes: yes - storage format changes: no - interface changes: yes Details: A usable embedding is a list of the expected length whose every element is numeric. VectorValidator states that rule once for the whole engine. Elements are inspected individually when the list's item type is ANY because a heterogeneous list declares nothing about any particular element. isvector(v) and isvector(v, dimension) surface the rule to SQL++ so that a user can ask which records an index will skip. Null and missing propagate as they do for is_array and the rest of the is_* family. A WHERE clause cannot tell either apart from false. An empty list is never a vector because there is nothing to index. Ext-ref: MB-73666 Co-Authored-By: Claude Opus 5 <[email protected]> Change-Id: I8631abc61ba7c15c53ec02ec5646d7bf92711eea --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/isvector/isvector.1.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/types/isvector/isvector.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java A asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/VectorValidator.java A asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IsVectorDescriptor.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java 8 files changed, 314 insertions(+), 0 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/53/21653/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/isvector/isvector.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/isvector/isvector.1.query.sqlpp new file mode 100644 index 0000000..f6ec24e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/types/isvector/isvector.1.query.sqlpp @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// isvector(v) and isvector(v, dimension). Null and missing propagate, matching the rest of the +// is_* family, so "k" is null and "l" is absent rather than both being false. +{ + "a_ok": isvector([0.2, 0.1, 0.4, 0.3]), + "b_ok_dim": isvector([0.2, 0.1, 0.4, 0.3], 4), + "c_mixed": isvector([0.2, 0.1, "s", 0.3], 4), + "d_all_string": isvector(["a", "b", "c", "d"], 4), + "e_null_elem": isvector([0.2, 0.1, null, 0.3], 4), + "f_nested": isvector([0.2, 0.1, [1], 0.3], 4), + "g_short": isvector([0.2, 0.1, 0.3], 4), + "h_empty_dim": isvector([], 4), + "i_empty_nodim": isvector([]), + "j_not_list": isvector("x"), + "k_null": isvector(null), + "l_missing": isvector(missing), + "m_integers": isvector([1, 2, 3], 3), + "n_underscore": is_vector([1, 2, 3], 3) +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/types/isvector/isvector.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/types/isvector/isvector.1.adm new file mode 100644 index 0000000..c9edfb0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/types/isvector/isvector.1.adm @@ -0,0 +1 @@ +{ "a_ok": true, "b_ok_dim": true, "c_mixed": false, "d_all_string": false, "e_null_elem": false, "f_nested": false, "g_short": false, "h_empty_dim": false, "i_empty_nodim": false, "j_not_list": false, "k_null": null, "m_integers": true, "n_underscore": true } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index a635411..4985d75 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -14980,6 +14980,11 @@ </compilation-unit> </test-case> <test-case FilePath="types"> + <compilation-unit name="isvector"> + <output-dir compare="Text">isvector</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="types"> <compilation-unit name="isatomic"> <output-dir compare="Text">isatomic</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java index 08776c5..d3d8ae1 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/CommonFunctionMapUtil.java @@ -106,6 +106,7 @@ addFunctionMapping("isstr", "is-string"); // isstr, internal: is-string addFunctionMapping("is_str", "is-string"); // is_str, internal: is-string addFunctionMapping("isarray", "is-array"); // isarray, internal: is-array + addFunctionMapping("isvector", "is-vector"); // isvector, internal: is-vector addFunctionMapping("ismultiset", "is-multiset"); // ismultiset, internal: is-multiset addFunctionMapping("isobject", "is-object"); // isobject, internal: is-object addFunctionMapping("isobj", "is-object"); // isobj, internal: is-object diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index 295f6e3..d125d14 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -1341,6 +1341,8 @@ public static final FunctionIdentifier IS_NUMBER = FunctionConstants.newAsterix("is-number", 1); public static final FunctionIdentifier IS_STRING = FunctionConstants.newAsterix("is-string", 1); public static final FunctionIdentifier IS_ARRAY = FunctionConstants.newAsterix("is-array", 1); + public static final FunctionIdentifier IS_VECTOR = + FunctionConstants.newAsterix("is-vector", FunctionIdentifier.VARARGS); public static final FunctionIdentifier IS_OBJECT = FunctionConstants.newAsterix("is-object", 1); public static final FunctionIdentifier IS_MULTISET = FunctionConstants.newAsterix("is-multiset", 1); public static final FunctionIdentifier GET_TYPE = FunctionConstants.newAsterix("get-type", 1); @@ -1441,6 +1443,7 @@ addFunction(IS_NUMBER, ABooleanTypeComputer.INSTANCE, true); addFunction(IS_STRING, ABooleanTypeComputer.INSTANCE, true); addFunction(IS_ARRAY, ABooleanTypeComputer.INSTANCE, true); + addFunction(IS_VECTOR, ABooleanTypeComputer.INSTANCE, true); addFunction(IS_OBJECT, ABooleanTypeComputer.INSTANCE, true); addFunction(IS_MULTISET, ABooleanTypeComputer.INSTANCE, true); addFunction(NOT, ABooleanTypeComputer.INSTANCE, true); diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/VectorValidator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/VectorValidator.java new file mode 100644 index 0000000..5a83509 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/VectorValidator.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.common; + +import java.io.IOException; + +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.EnumDeserializer; +import org.apache.asterix.om.types.hierachy.ATypeHierarchy; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.util.annotations.AiProvenance; + +/** + * The single definition of what makes a value a usable vector: a list, of the expected length, whose every + * element is numeric. + * <p> + * Elements are checked individually when the list's item type is {@code ANY}, since a heterogeneous list + * declares nothing about any particular element: {@code [0.2, 0.1, "s", 0.3]} is not a usable vector. + * <p> + * This class is deliberately <b>total</b>: it never throws, for any input. {@code isvector} is registered as + * a TOTAL function, and CLUSTER BY's guard depends on that to keep the optimizer from reordering it into + * something that can fail. + * <p> + * Instances hold scratch state and are <b>not</b> thread-safe; create one per operator instance, as + * {@code KMeansUtils} is used. + */ +@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.ASSISTED) +public final class VectorValidator { + + /** Why a value is or is not a usable vector. Callers that only need a boolean compare against {@link #OK}. */ + public enum Verdict { + /** A list of the expected length whose every element is numeric. */ + OK, + /** Absent, null or missing. Distinct from a malformed value: the field simply carries nothing. */ + UNKNOWN, + /** Present, but not a list at all: a string, a number, an object. */ + NOT_A_LIST, + /** A list, but empty or not of the expected length. See {@link #getObservedDimension()}. */ + WRONG_DIMENSION, + /** A list of the right length holding at least one non-numeric element. */ + BAD_ELEMENT + } + + private final ListAccessor listAccessor = new ListAccessor(); + private final IPointable item = new VoidPointable(); + private final ArrayBackedValueStorage storage = new ArrayBackedValueStorage(); + + private int observedDimension = -1; + + /** + * Classifies the value at {@code offset}. + * + * @param expectedDimension the declared dimension, or a non-positive value to accept any non-zero + * length. An empty list is {@link Verdict#WRONG_DIMENSION} either way. + */ + public Verdict validate(byte[] data, int offset, int length, int expectedDimension) { + observedDimension = -1; + if (length == 0) { + return Verdict.UNKNOWN; + } + ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(data[offset]); + if (typeTag == ATypeTag.MISSING || typeTag == ATypeTag.NULL || typeTag == ATypeTag.SYSTEM_NULL) { + return Verdict.UNKNOWN; + } + if (typeTag == null || !typeTag.isListType()) { + return Verdict.NOT_A_LIST; + } + try { + listAccessor.reset(data, offset); + int size = listAccessor.size(); + // An empty list is never a usable vector, with or without a declared dimension: there is nothing + // to index. Reported as a dimension problem, with an observed length of 0. + if (size == 0 || (expectedDimension > 0 && size != expectedDimension)) { + observedDimension = size; + return Verdict.WRONG_DIMENSION; + } + return validateElements(size); + } catch (HyracksDataException e) { + // Tagged as a list but unreadable. Not usable, and this class does not throw. + return Verdict.BAD_ELEMENT; + } + } + + /** The length actually found, valid only after {@link Verdict#WRONG_DIMENSION}; otherwise -1. */ + public int getObservedDimension() { + return observedDimension; + } + + /** + * A homogeneous list is settled by its item type alone, which is the common case and stays O(1). Only a + * heterogeneous list ({@code ANY}) has to be walked, because there the item type says nothing about any + * particular element. That is the gap that let {@code [0.2, 0.1, "s", 0.3]} through everywhere. + */ + private Verdict validateElements(int size) throws HyracksDataException { + ATypeTag itemTypeTag = listAccessor.getItemType(); + // A null item type means the list does not declare one, which tells us nothing about any particular + // element; walk them, exactly as for a heterogeneous ANY list. + if (itemTypeTag != null && itemTypeTag != ATypeTag.ANY) { + return isNumeric(itemTypeTag) ? Verdict.OK : Verdict.BAD_ELEMENT; + } + for (int i = 0; i < size; i++) { + try { + listAccessor.getOrWriteItem(i, item, storage); + } catch (IOException e) { + return Verdict.BAD_ELEMENT; + } + ATypeTag elementTag = + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(item.getByteArray()[item.getStartOffset()]); + if (!isNumeric(elementTag)) { + return Verdict.BAD_ELEMENT; + } + } + return Verdict.OK; + } + + private static boolean isNumeric(ATypeTag typeTag) { + return typeTag != null && ATypeHierarchy.getTypeDomain(typeTag) == ATypeHierarchy.Domain.NUMERIC; + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IsVectorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IsVectorDescriptor.java new file mode 100644 index 0000000..737f49c --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/IsVectorDescriptor.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.functions; + +import java.io.DataOutput; + +import org.apache.asterix.common.annotations.MissingNullInOutFunction; +import org.apache.asterix.dataflow.data.nontagged.serde.AObjectSerializerDeserializer; +import org.apache.asterix.om.base.ABoolean; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.types.hierachy.ATypeHierarchy; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.asterix.runtime.evaluators.common.VectorValidator; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IEvaluatorContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference; +import org.apache.hyracks.util.annotations.AiProvenance; + +/** + * {@code isvector(v)} and {@code isvector(v, dimension)}: true when {@code v} is a list whose every element + * is numeric and, given a dimension, whose length matches it. + * <p> + * This is the user-facing statement of the rule the vector index enforces internally, so that + * <blockquote>the set of rows a VTREE index contains is exactly the set of rows for which + * {@code isvector(field, dimension)} is true.</blockquote> + * The index build applies this same predicate, which makes the contract true by construction and lets a user + * ask which of their records were skipped, and why: + * + * <pre> + * SELECT COUNT(*) FROM ds WHERE NOT isvector(ds.emb, 384); + * SELECT DISTINCT array_count(ds.emb) FROM ds; + * </pre> + * <p> + * Null and missing propagate, matching {@code is_array} and the rest of the {@code is_*} family, so + * {@code isvector(null)} is {@code null} rather than {@code false}. In a WHERE clause that is + * indistinguishable from {@code false}, since neither is TRUE. + * <p> + * The function is <b>total</b>, since {@link VectorValidator} never throws, which CLUSTER BY's usable-vector + * guard relies on. + */ +@MissingNullInOutFunction +@AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_CLI, contributionKind = AiProvenance.ContributionKind.ASSISTED) +public class IsVectorDescriptor extends AbstractScalarFunctionDynamicDescriptor { + + public static final IFunctionDescriptorFactory FACTORY = IsVectorDescriptor::new; + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException { + return new IsVectorEvaluator(args, ctx); + } + }; + } + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.IS_VECTOR; + } + + private static final class IsVectorEvaluator implements IScalarEvaluator { + + private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage(); + private final DataOutput out = resultStorage.getDataOutput(); + private final IPointable valuePtr = new VoidPointable(); + private final IPointable dimensionPtr = new VoidPointable(); + private final IScalarEvaluator valueEval; + private final IScalarEvaluator dimensionEval; + private final VectorValidator validator = new VectorValidator(); + + private IsVectorEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx) throws HyracksDataException { + valueEval = args[0].createScalarEvaluator(ctx); + dimensionEval = args.length > 1 ? args[1].createScalarEvaluator(ctx) : null; + } + + @Override + public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { + valueEval.evaluate(tuple, valuePtr); + if (dimensionEval == null) { + if (PointableHelper.checkAndSetMissingOrNull(result, valuePtr)) { + return; + } + } else { + dimensionEval.evaluate(tuple, dimensionPtr); + if (PointableHelper.checkAndSetMissingOrNull(result, valuePtr, dimensionPtr)) { + return; + } + } + // Any dimension when none was given; getIntegerValue rejects a non-numeric second argument. + int dimension = dimensionEval == null ? -1 + : ATypeHierarchy.getIntegerValue(BuiltinFunctions.IS_VECTOR.getName(), 1, + dimensionPtr.getByteArray(), dimensionPtr.getStartOffset()); + boolean isVector = validator.validate(valuePtr.getByteArray(), valuePtr.getStartOffset(), + valuePtr.getLength(), dimension) == VectorValidator.Verdict.OK; + resultStorage.reset(); + AObjectSerializerDeserializer.INSTANCE.serialize(isVector ? ABoolean.TRUE : ABoolean.FALSE, out); + result.set(resultStorage); + } + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java index 95e0d58..3e0498d 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java @@ -438,6 +438,7 @@ import org.apache.asterix.runtime.evaluators.functions.IsTimeDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsUUIDDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor; +import org.apache.asterix.runtime.evaluators.functions.IsVectorDescriptor; import org.apache.asterix.runtime.evaluators.functions.LenDescriptor; import org.apache.asterix.runtime.evaluators.functions.NotDescriptor; import org.apache.asterix.runtime.evaluators.functions.NumericACosDescriptor; @@ -1376,6 +1377,7 @@ // Type functions. fc.add(GetTypeDescriptor.FACTORY); fc.add(IsArrayDescriptor.FACTORY); + fc.add(IsVectorDescriptor.FACTORY); fc.add(IsAtomicDescriptor.FACTORY); fc.add(IsBooleanDescriptor.FACTORY); fc.add(IsNumberDescriptor.FACTORY); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21653?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I8631abc61ba7c15c53ec02ec5646d7bf92711eea Gerrit-Change-Number: 21653 Gerrit-PatchSet: 1 Gerrit-Owner: Hongyu Shi <[email protected]>
