>From Ali Alsuliman <[email protected]>: Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21621?usp=email )
Change subject: [ASTERIXDB-3817][COMP][MTD] Vector index seed as a WITH parameter ...................................................................... [ASTERIXDB-3817][COMP][MTD] Vector index seed as a WITH parameter Replace the compiler.vector.trainseed request parameter with a `seed` WITH parameter, persisted with the rest of the vector index parameters. A request parameter could not be recorded anywhere, so a build was reproducible only within the request that created the index. It also seeded k-means alone -- the train-list sample drew its own -- so a sampled train list (train_list_fraction < 1.0) was irreproducible no matter what k-means got. One seed now drives both. `seed` is optional in the WITH clause and materialized at DDL time, which is what makes it persistable: the Metadata.`Index` record is written before the creation job is built. A configuration that still reaches the builder without one is re-seeded with a warning rather than rejected, so a record that lost the field stays loadable. compiler.vector.trainseed was never a registered configurable parameter, so it only ever worked ahead of a DDL statement; the same SET ahead of a query failed as an unsupported query parameter. asterix-om's log4j-api dependency moves from test to compile scope for that warning. log4j-core was already at compile scope there. Ext-ref: MB-73194 Co-Authored-By: Claude Opus 5 <[email protected]> Change-Id: I1c2ede3846f2ed7f4802d969922207d19775beb1 --- M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-dimension-mismatch/create-index-vtree-dimension-mismatch.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-meta-pk/create-index-vtree-meta-pk.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.4.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.6.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.7.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.adm M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryVectorOperationsHelper.java M asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/VectorIndexParametersTupleTranslatorTest.java M asterixdb/asterix-om/pom.xml M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/vector/VectorIndexParameters.java 23 files changed, 233 insertions(+), 76 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/21/21621/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp index a20ea44..80ae61f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-composite-pk/create-index-vtree-composite-pk.4.ddl.sqlpp @@ -19,11 +19,8 @@ USE testcpk; -// Pin the k-means training RNG so index creation is reproducible run-to-run. -SET `compiler.vector.trainseed` "42"; - CREATE INDEX idx_cpk_emb ON MovieCpk(embedding VECTOR) INCLUDE (year) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-dimension-mismatch/create-index-vtree-dimension-mismatch.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-dimension-mismatch/create-index-vtree-dimension-mismatch.4.ddl.sqlpp index 23ff428..2e47142 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-dimension-mismatch/create-index-vtree-dimension-mismatch.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-dimension-mismatch/create-index-vtree-dimension-mismatch.4.ddl.sqlpp @@ -19,11 +19,9 @@ USE test; -SET `compiler.vector.trainseed` "42"; - // The data is 4-dimensional, so no sampled vector can be indexed under dimension 8. CREATE INDEX idx_emb ON MovieSmall(embedding VECTOR) TYPE VTREE - WITH { "dimension": 8, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 8, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp index 069c557..7eec251 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-glove/create-index-vtree-glove.4.ddl.sqlpp @@ -19,7 +19,6 @@ USE testg; -SET `compiler.vector.trainseed` "42"; CREATE INDEX idx_emb ON GloveReal(embedding VECTOR) TYPE VTREE - WITH { "dimension": 100, "similarity": "cosine", "num_clusters": 4, "train_list_fraction": 1.0 } + WITH { "dimension": 100, "similarity": "cosine", "num_clusters": 4, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp index e00e640..3fbfdcd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-filter/create-index-vtree-include-filter.4.ddl.sqlpp @@ -17,10 +17,9 @@ * under the License. */ USE test; -SET `compiler.vector.trainseed` "42"; CREATE INDEX idx_emb ON MovieSmall(embedding VECTOR) INCLUDE (year) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp index ca3b9cb..ca1dc2c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-include-nested/create-index-vtree-include-nested.4.ddl.sqlpp @@ -18,10 +18,9 @@ */ USE test; -SET `compiler.vector.trainseed` "42"; CREATE INDEX idx_emb ON MovieSmall(embedding VECTOR) INCLUDE (info.year) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-meta-pk/create-index-vtree-meta-pk.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-meta-pk/create-index-vtree-meta-pk.4.ddl.sqlpp index e2300f2..fe652da 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-meta-pk/create-index-vtree-meta-pk.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-meta-pk/create-index-vtree-meta-pk.4.ddl.sqlpp @@ -19,11 +19,8 @@ USE test; -// Pin the k-means training RNG so index creation is reproducible run-to-run. -SET `compiler.vector.trainseed` "42"; - CREATE INDEX idx_emb ON VecMeta(embedding VECTOR) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.4.ddl.sqlpp index af0a872..bab128d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.4.ddl.sqlpp @@ -19,8 +19,6 @@ USE test; -SET `compiler.vector.trainseed` "42"; - // Every WITH parameter is given a non-default value, and one INCLUDE path is nested, so the // expected record below shows the actual persisted value of each rather than a default that // would also be produced by dropping the field. @@ -34,6 +32,7 @@ "quantization": "SQ4", "num_clusters": 2, "train_list_fraction": 1.0, + "seed": 42, "epsilon": 0.5, "cross_pollination_m": 3, "rng_factor": 1.5 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.6.ddl.sqlpp new file mode 100644 index 0000000..9f55ec5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.6.ddl.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +DROP INDEX MovieSmall.idx_emb; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.7.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.7.ddl.sqlpp new file mode 100644 index 0000000..cbfb58d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.7.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +// Same index with `seed` omitted. The seed has no constant default, so DDL draws one and +// persists it; the query below checks that a seed was recorded rather than left absent. +CREATE INDEX idx_emb + ON MovieSmall(embedding VECTOR) + TYPE VTREE + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.query.sqlpp new file mode 100644 index 0000000..85d7b03 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// An omitted `seed` is materialized at DDL time, so the persisted record carries the seed the +// build actually used. The value itself is random, so only its presence and type can be asserted. +SELECT VALUE IS_NUMBER(i.seed) +FROM Metadata.`Index` i +WHERE i.DataverseName = "test" AND i.DatasetName = "MovieSmall" AND i.IndexStructure = "VTREE"; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp index c978239..d51f39e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-movie/create-index-vtree-movie.4.ddl.sqlpp @@ -19,7 +19,6 @@ USE testm; -SET `compiler.vector.trainseed` "42"; CREATE INDEX idx_emb ON MovieReal(embedding VECTOR) TYPE VTREE - WITH { "dimension": 384, "similarity": "l2_squared", "num_clusters": 4, "train_list_fraction": 1.0 } + WITH { "dimension": 384, "similarity": "l2_squared", "num_clusters": 4, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp index 29219d4..646da6d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-nested/create-index-vtree-nested.4.ddl.sqlpp @@ -19,10 +19,9 @@ USE test; // Nested key path (meta.embedding) and nested INCLUDE path (meta.year). -SET `compiler.vector.trainseed` "42"; CREATE INDEX idx_emb ON MovieNested(meta.embedding VECTOR) INCLUDE (meta.year) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp index bb68182..6578c39 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-open-undeclared/create-index-vtree-open-undeclared.4.ddl.sqlpp @@ -18,10 +18,9 @@ */ USE test; -SET `compiler.vector.trainseed` "42"; // Index on the UNDECLARED 'embedding' field of the OPEN type; this builds like any declared vector field. CREATE INDEX idx_emb ON MovieOpen(embedding VECTOR) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp index 0d34c95..7855251 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-similarity-normalize/create-index-vtree-similarity-normalize.4.ddl.sqlpp @@ -18,11 +18,10 @@ */ USE test; -SET `compiler.vector.trainseed` "42"; // similarity given as the alias "L2" (not the canonical "euclidean"); validateSimilarity must accept it // via VectorSimilarityMetric.fromAlias and normalize the stored value to the canonical spelling. CREATE INDEX idx_emb ON MovieSmall(embedding VECTOR) TYPE VTREE - WITH { "dimension": 4, "similarity": "L2", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "L2", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp index 17ea81c..d8d6d29 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree/create-index-vtree.4.ddl.sqlpp @@ -19,11 +19,8 @@ USE test; -// Pin the k-means training RNG so index creation is reproducible run-to-run. -SET `compiler.vector.trainseed` "42"; - CREATE INDEX idx_emb ON MovieSmall(embedding VECTOR) TYPE VTREE - WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0 } + WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "train_list_fraction": 1.0, "seed": 42 } EXCLUDE UNKNOWN KEY; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm index 2c58ba7..217e8f6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm @@ -1 +1 @@ -{ "DataverseName": "test", "DatasetName": "MovieSmall", "IndexName": "idx_emb", "IndexStructure": "VTREE", "SearchKey": [ [ "embedding" ] ], "IsPrimary": false, "PendingOp": 0, "dimension": 4, "similarity": "cosine", "quantization": "SQ4", "train_list_fraction": 1.0, "epsilon": 0.5, "num_clusters": 2, "cross_pollination_m": 3, "rng_factor": 1.5, "IncludeFields": [ [ "title" ], [ "info", "year" ] ] } \ No newline at end of file +{ "DataverseName": "test", "DatasetName": "MovieSmall", "IndexName": "idx_emb", "IndexStructure": "VTREE", "SearchKey": [ [ "embedding" ] ], "IsPrimary": false, "PendingOp": 0, "dimension": 4, "similarity": "cosine", "quantization": "SQ4", "train_list_fraction": 1.0, "epsilon": 0.5, "num_clusters": 2, "cross_pollination_m": 3, "rng_factor": 1.5, "seed": 42, "IncludeFields": [ [ "title" ], [ "info", "year" ] ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.adm new file mode 100644 index 0000000..27ba77d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.8.adm @@ -0,0 +1 @@ +true diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index d5cc336..49f604f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -309,8 +309,6 @@ public static final String COMPILER_PARQUET_FILESPLITS_KEY = Option.COMPILER_PARQUET_FILESPLITS.ini(); public static final String COMPILER_HDFS_SPLIT_PARALLELISM_KEY = Option.COMPILER_HDFS_SPLIT_PARALLELISM.ini(); public static final String COMPILER_VECTOR_K_MULTIPLIER_KEY = "compiler.vector.kmultiplier"; - // Seeds the k-means training RNG of vector index creation; set per request for deterministic tests. - public static final String COMPILER_VECTOR_TRAINSEED_KEY = "compiler.vector.trainseed"; public CompilerProperties(PropertiesAccessor accessor) { super(accessor); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java index 670e726..abc1ebc 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java @@ -26,12 +26,14 @@ import static org.apache.asterix.om.vector.VectorIndexParameters.NUM_CLUSTERS; import static org.apache.asterix.om.vector.VectorIndexParameters.QUANTIZATION; import static org.apache.asterix.om.vector.VectorIndexParameters.RNG_FACTOR; +import static org.apache.asterix.om.vector.VectorIndexParameters.SEED; import static org.apache.asterix.om.vector.VectorIndexParameters.SIMILARITY; import static org.apache.asterix.om.vector.VectorIndexParameters.TRAIN_LIST_FRACTION; import java.util.Arrays; import java.util.Locale; import java.util.OptionalInt; +import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; import org.apache.asterix.common.exceptions.AsterixException; @@ -48,6 +50,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.vector.VectorIndexParameters; import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.util.annotations.AiProvenance; /** * Validates the {@code WITH} clause of a {@code CREATE INDEX ... TYPE VTREE} statement and turns it into the @@ -101,6 +104,7 @@ validateNumClusters(node).ifPresent(builder::setNumClusters); builder.setCrossPollinationM(validateCrossPollinationM(node)); builder.setRngFactor(validateRngFactor(node)); + builder.setSeed(validateSeed(node)); try { return builder.build(); } catch (AsterixException e) { @@ -276,6 +280,28 @@ return value; } + /** + * Validates {@code seed}, shared by the train-list sample and the k-means RNG. Every {@code long} is a + * usable seed, so the only check is the type. + * <p> + * Unlike the other optional parameters this one has no constant default: when the user does not give a + * seed we draw one here and persist it, so the index records the seed its build actually used and can be + * rebuilt identically. Drawing it at DDL time rather than at job-generation time is what makes it + * persistable — the {@code Metadata.Index} record is written before the creation job is built. + */ + @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED) + private static long validateSeed(AdmObjectNode node) throws CompilationException { + IAdmNode seedNode = node.get(SEED); + if (seedNode == null) { + return ThreadLocalRandom.current().nextLong(); + } + if (seedNode.getType() != ATypeTag.BIGINT) { + throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED, + "Invalid `seed` parameter value. It must be an integer."); + } + return ((AdmBigIntNode) seedNode).get(); + } + private static double parseDoubleOrBigInt(IAdmNode n, String errorMsg) throws CompilationException { switch (n.getType()) { case DOUBLE: diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryVectorOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryVectorOperationsHelper.java index 67c9d4a..ab47484 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryVectorOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryVectorOperationsHelper.java @@ -26,7 +26,6 @@ import java.util.UUID; import org.apache.asterix.common.cluster.PartitioningProperties; -import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.OptimizationConfUtil; import org.apache.asterix.common.context.ITransactionSubsystemProvider; @@ -107,13 +106,9 @@ import org.apache.hyracks.storage.common.IResourceFactory; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory; -import org.apache.hyracks.util.annotations.AiProvenance; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; public class SecondaryVectorOperationsHelper extends SecondaryTreeIndexOperationsHelper { - private static final Logger LOGGER = LogManager.getLogger(); private RecordDescriptor recordDesc; private static final float DEFAULT_CONFIDENCE_INTERVAL = 0.99f; /** Minimum train-list sample size for static-structure build; below this after clamp → full scan. */ @@ -160,25 +155,6 @@ return sampleSize < TRAIN_LIST_MIN_SAMPLE_SIZE; } - /** - * The seed shared by the train-list sample and the k-means RNG: the request-level - * {@code compiler.vector.trainseed} when set (so an index build is reproducible), otherwise a fresh - * {@code nanoTime()} seed. - */ - @AiProvenance(agent = AiProvenance.Agent.CLAUDE_OPUS_5, tool = AiProvenance.Tool.CLAUDE_CODE_UI, contributionKind = AiProvenance.ContributionKind.GENERATED) - private long resolveTrainSeed() { - Object trainSeedCfg = metadataProvider.getConfig().get(CompilerProperties.COMPILER_VECTOR_TRAINSEED_KEY); - if (trainSeedCfg != null) { - try { - return Long.parseLong(String.valueOf(trainSeedCfg).trim()); - } catch (NumberFormatException e) { - LOGGER.warn("Invalid {} '{}', using a random seed", CompilerProperties.COMPILER_VECTOR_TRAINSEED_KEY, - trainSeedCfg); - } - } - return System.nanoTime(); - } - @Override public JobSpecification buildStaticStructureJobSpec() throws AlgebricksException { IDataFormat format = metadataProvider.getDataFormat(); @@ -206,14 +182,12 @@ // ============ SAMPLING OR FULL SCAN based on sampleSize threshold ============ // Extract sampling parameters from WITH clause (train_list_fraction only; validated at compile time) double trainListFraction = indexDetails.getVectorParameters().getTrainListFraction(); - // Seed for BOTH the train-list sample and the k-means RNG: overridable per request - // (SET `compiler.vector.trainseed` "42") so CI / regression tests get reproducible centroids, and a - // fresh random seed otherwise. The sample seed must honour the same override, otherwise a sampled - // train list (train_list_fraction < 1.0) makes the whole index build irreproducible no matter what - // the k-means seed is. sample_seed is not a supported WITH field - // (it is not declared in VectorIndexParameters), so this is the only channel. - long trainSeed = resolveTrainSeed(); - long sampleSeed = trainSeed; + // One seed drives BOTH the train-list sample and the k-means RNG, so a build over the same data with + // the same partitioning and the same seed produces the same centroids. Seeding only k-means would + // leave a sampled train list (train_list_fraction < 1.0) irreproducible no matter what k-means got. + // The seed was materialized at DDL time, so it is already persisted in this index's metadata: the + // build is reproducible from what the catalog records, not just within a single request. + long seed = indexDetails.getVectorParameters().getSeed(); // Retrieve cardinality from sample index metadata (needed for fraction-based sample size) Index sampleIndex = metadataProvider.findSampleIndex(dataset.getDatabaseName(), dataset.getDataverseName(), @@ -248,7 +222,7 @@ } else { int sampleCardinalityPerPartition = Math.max(1, sampleSize / numPartitions); targetOp = DatasetUtil.createSampleScanOp(spec, metadataProvider, dataset, sampleCardinalityPerPartition, - sampleSeed, projectorFactory); + seed, projectorFactory); } spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0); @@ -307,7 +281,7 @@ HierarchicalKMeansPlusPlusCentroidsOperatorDescriptor candidates = new HierarchicalKMeansPlusPlusCentroidsOperatorDescriptor(spec, hierarchicalRecDesc, secondaryRecDesc, sampleUUID, tupleCountUUID, new ColumnAccessEvalFactory(0), K, maxScalableKmeansIter, - distanceMetric, vectorDimension, trainSeed); + distanceMetric, vectorDimension, seed); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, candidates, primaryPartitionConstraint); targetOp = candidates; diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/VectorIndexParametersTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/VectorIndexParametersTupleTranslatorTest.java index 2432ee1..78ffa7f 100644 --- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/VectorIndexParametersTupleTranslatorTest.java +++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/VectorIndexParametersTupleTranslatorTest.java @@ -80,7 +80,7 @@ VectorIndexParameters written = VectorIndexParameters.builder().setDimension(128).setSimilarity(VectorSimilarityMetric.EUCLIDEAN) .setQuantization(VectorQuantization.SQ4).setTrainListFraction(0.375).setEpsilon(0.625) - .setNumClusters(7).setCrossPollinationM(3).setRngFactor(1.5).build(); + .setNumClusters(7).setCrossPollinationM(3).setRngFactor(1.5).setSeed(-9876543210L).build(); VectorIndexParameters readBack = roundTrip(written); @@ -95,6 +95,9 @@ Assert.assertEquals(OptionalInt.of(7), readBack.getNumClusters()); Assert.assertEquals(3, readBack.getCrossPollinationM()); Assert.assertEquals(1.5, readBack.getRngFactor(), 0.0); + // A seed is a full 64-bit value, and a negative one is as valid as any other: storing it as an INTEGER + // or reading it back through an int would corrupt exactly the seeds the RNG is most likely to draw. + Assert.assertEquals(-9876543210L, readBack.getSeed()); } /** @@ -104,13 +107,16 @@ */ @Test public void absentParametersReadBackAsDefaults() throws AlgebricksException, IOException { - VectorIndexParameters minimal = - VectorIndexParameters.builder().setDimension(4).setSimilarity(VectorSimilarityMetric.COSINE).build(); + VectorIndexParameters minimal = VectorIndexParameters.builder().setDimension(4) + .setSimilarity(VectorSimilarityMetric.COSINE).setSeed(0L).build(); VectorIndexParameters readBack = roundTrip(minimal); Assert.assertEquals(minimal, readBack); Assert.assertEquals(OptionalInt.empty(), readBack.getNumClusters()); + // 0 is a seed like any other, not an "unset" marker: it must survive the round trip rather than being + // treated as absent and redrawn. + Assert.assertEquals(0L, readBack.getSeed()); Assert.assertEquals(VectorIndexParameters.DEFAULT_TRAIN_LIST_FRACTION, readBack.getTrainListFraction(), 0.0); Assert.assertEquals(VectorIndexParameters.DEFAULT_EPSILON, readBack.getEpsilon(), 0.0); Assert.assertEquals(VectorIndexParameters.DEFAULT_CROSS_POLLINATION_M, readBack.getCrossPollinationM()); @@ -130,6 +136,30 @@ } /** + * seed is the one parameter with neither a constant default nor a mandatory-set check: an unset one is + * drawn, so a persisted record that lost the field still loads. Two seedless builders must not agree, or + * the "draw" is really a fixed default and every such index would share one seed. + */ + @Test + public void unsetSeedIsDrawnRatherThanRejected() throws AlgebricksException { + // Independent draws collide with probability 2^-64. + Assert.assertNotEquals(seedlessBuilder().build().getSeed(), seedlessBuilder().build().getSeed()); + } + + /** Rebuilding one builder repeats its drawn seed, so two reads of the same record cannot disagree. */ + @Test + public void aDrawnSeedIsStableAcrossRebuilds() throws AlgebricksException { + VectorIndexParameters.Builder builder = seedlessBuilder(); + + Assert.assertEquals(builder.build().getSeed(), builder.build().getSeed()); + Assert.assertEquals(builder.build(), builder.build()); + } + + private static VectorIndexParameters.Builder seedlessBuilder() { + return VectorIndexParameters.builder().setDimension(4).setSimilarity(VectorSimilarityMetric.COSINE); + } + + /** * Every instance field must have a matching entry in {@code NAMES}. This cannot prove {@code writeFields} * and {@code readFields} were updated, but it fails the moment a parameter is added, which is the point at * which both need a line — and {@link #everyParameterRoundTrips} then catches a missing one. diff --git a/asterixdb/asterix-om/pom.xml b/asterixdb/asterix-om/pom.xml index ec805b8..dd1d9d28 100644 --- a/asterixdb/asterix-om/pom.xml +++ b/asterixdb/asterix-om/pom.xml @@ -141,7 +141,6 @@ <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>it.unimi.dsi</groupId> diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/vector/VectorIndexParameters.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/vector/VectorIndexParameters.java index 90dc817..2e5032d 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/vector/VectorIndexParameters.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/vector/VectorIndexParameters.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Objects; import java.util.OptionalInt; +import java.util.concurrent.ThreadLocalRandom; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.common.exceptions.AsterixException; @@ -45,6 +46,8 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.util.annotations.AiProvenance; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * The validated configuration of a {@code CREATE INDEX ... TYPE VTREE} index: one typed field per accepted @@ -71,6 +74,8 @@ private static final long serialVersionUID = 1L; + private static final Logger LOGGER = LogManager.getLogger(); + public static final String DIMENSION = "dimension"; public static final String SIMILARITY = "similarity"; public static final String QUANTIZATION = "quantization"; @@ -79,6 +84,7 @@ public static final String NUM_CLUSTERS = "num_clusters"; public static final String CROSS_POLLINATION_M = "cross_pollination_m"; public static final String RNG_FACTOR = "rng_factor"; + public static final String SEED = "seed"; public static final VectorQuantization DEFAULT_QUANTIZATION = VectorQuantization.SQ8; public static final double DEFAULT_TRAIN_LIST_FRACTION = 0.1; @@ -93,7 +99,7 @@ * written for a given configuration are stable. Kept in step with the two serde methods by hand. */ private static final List<String> NAMES = List.of(DIMENSION, SIMILARITY, QUANTIZATION, TRAIN_LIST_FRACTION, EPSILON, - NUM_CLUSTERS, CROSS_POLLINATION_M, RNG_FACTOR); + NUM_CLUSTERS, CROSS_POLLINATION_M, RNG_FACTOR, SEED); private final int dimension; private final VectorSimilarityMetric similarity; @@ -104,6 +110,7 @@ private final Integer numClusters; private final int crossPollinationM; private final double rngFactor; + private final long seed; private VectorIndexParameters(Builder builder) { this.dimension = builder.dimension; @@ -114,6 +121,7 @@ this.numClusters = builder.numClusters; this.crossPollinationM = builder.crossPollinationM; this.rngFactor = builder.rngFactor; + this.seed = builder.seed; } public static Builder builder() { @@ -178,6 +186,19 @@ return rngFactor; } + /** + * The seed shared by the train-list sample and the k-means RNG, so a build over the same data with the + * same partitioning and the same seed produces the same centroids. + * <p> + * Optional in the {@code WITH} clause but never absent from a configuration: {@code VectorIndexDeclUtil} + * draws one when the user does not give one, which is what lets the seed be persisted alongside the + * parameters the user did write, and {@link Builder#build()} draws one for the case that slips past even + * that. Every value is a legal seed, {@code 0} included. + */ + public long getSeed() { + return seed; + } + /** Every accepted parameter name, in persist order. */ public static List<String> names() { return NAMES; @@ -210,6 +231,7 @@ } writer.writeInt(CROSS_POLLINATION_M, crossPollinationM); writer.writeDouble(RNG_FACTOR, rngFactor); + writer.writeLong(SEED, seed); } /** @@ -271,6 +293,10 @@ if (rngFactor != null) { builder.setRngFactor(rngFactor); } + Long seed = reader.readLong(SEED); + if (seed != null) { + builder.setSeed(seed); + } return builder.build(); } @@ -286,13 +312,14 @@ && Objects.equals(quantization, other.quantization) && Double.compare(trainListFraction, other.trainListFraction) == 0 && Double.compare(epsilon, other.epsilon) == 0 && Objects.equals(numClusters, other.numClusters) - && crossPollinationM == other.crossPollinationM && Double.compare(rngFactor, other.rngFactor) == 0; + && crossPollinationM == other.crossPollinationM && Double.compare(rngFactor, other.rngFactor) == 0 + && seed == other.seed; } @Override public int hashCode() { return Objects.hash(dimension, similarity, quantization, trainListFraction, epsilon, numClusters, - crossPollinationM, rngFactor); + crossPollinationM, rngFactor, seed); } @Override @@ -308,13 +335,15 @@ } sb.append(", ").append(CROSS_POLLINATION_M).append(": ").append(crossPollinationM); sb.append(", ").append(RNG_FACTOR).append(": ").append(rngFactor); + sb.append(", ").append(SEED).append(": ").append(seed); return sb.append(" }").toString(); } /** * Collects parameter values and produces an immutable {@link VectorIndexParameters}. Unset optional * parameters take their declared default; {@code dimension} and {@code similarity} have no default and - * must be set, so a built instance is always a usable configuration. + * must be set, so a built instance is always a usable configuration. {@code seed} has no default either, + * but an unset one is drawn rather than rejected, so it is never the reason a build fails. */ public static final class Builder { @@ -326,6 +355,8 @@ private Integer numClusters; private int crossPollinationM = DEFAULT_CROSS_POLLINATION_M; private double rngFactor = DEFAULT_RNG_FACTOR; + private long seed; + private boolean seedSet; private Builder() { } @@ -370,6 +401,12 @@ return this; } + public Builder setSeed(long seed) { + this.seed = seed; + this.seedSet = true; + return this; + } + public boolean hasDimension() { return dimension > 0; } @@ -379,6 +416,14 @@ } /** + * Tracked by a flag rather than a sentinel value, because every {@code long} — {@code 0} included — + * is a seed a caller may legitimately have asked for, so no value can stand in for "unset". + */ + private boolean hasSeed() { + return seedSet; + } + + /** * @throws AsterixException if a mandatory parameter was never set — at DDL time the validator reports * the missing key first, so reaching this means corrupt persisted metadata. */ @@ -391,6 +436,19 @@ throw new AsterixException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED, "Missing `" + SIMILARITY + "` parameter in the WITH clause"); } + if (!hasSeed()) { + // Unlike the two above, this is never the user's omission: DDL draws a seed when the WITH + // clause has none, so getting here means a persisted record that lost the field. Drawing a + // replacement keeps the index usable — nothing but reproducibility depends on the seed, and + // that is already gone, since the original is recoverable from nowhere else. Warn rather + // than fail, but do warn: silence would make a lost field indistinguishable from a build the + // user never pinned. + setSeed(ThreadLocalRandom.current().nextLong()); + LOGGER.warn( + "Vector index parameters carried no `{}`; drew {} instead. The seed of the " + + "original build is unrecoverable, so rebuilding will not reproduce its centroids.", + SEED, seed); + } return new VectorIndexParameters(this); } } @@ -413,6 +471,9 @@ private final ISerializerDeserializer<AInt32> int32Serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32); @SuppressWarnings("unchecked") + private final ISerializerDeserializer<AInt64> int64Serde = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64); + @SuppressWarnings("unchecked") private final ISerializerDeserializer<ADouble> doubleSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE); private final ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage(); @@ -429,6 +490,13 @@ recordBuilder.addField(fieldName, fieldValue); } + private void writeLong(String name, long value) throws HyracksDataException { + writeName(name); + fieldValue.reset(); + int64Serde.serialize(new AInt64(value), fieldValue.getDataOutput()); + recordBuilder.addField(fieldName, fieldValue); + } + private void writeDouble(String name, double value) throws HyracksDataException { writeName(name); fieldValue.reset(); @@ -484,6 +552,12 @@ return longValue.intValue(); } + /** @return the value widened to {@code long}, or {@code null} when the field is absent or not numeric. */ + private Long readLong(String name) { + IAObject value = read(name); + return value == null ? null : asLong(value.getType().getTypeTag(), value); + } + private Double readDouble(String name) { IAObject value = read(name); return value == null ? null : asDouble(value.getType().getTypeTag(), value); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21621?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I1c2ede3846f2ed7f4802d969922207d19775beb1 Gerrit-Change-Number: 21621 Gerrit-PatchSet: 1 Gerrit-Owner: Ali Alsuliman <[email protected]>
