This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7998 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit ed20fc94171bb8bb303abc24d1d74b7cb5ea8282 Author: Andrew Purtell <[email protected]> AuthorDate: Fri Sep 18 14:11:21 2026 -0700 PHOENIX-7998 Phase 5: Vector index DDL and grammar Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --- phoenix-core-client/src/main/antlr3/PhoenixSQL.g | 17 + .../phoenix/compile/CreateIndexCompiler.java | 74 +++ .../apache/phoenix/parse/CreateIndexStatement.java | 73 +++ .../org/apache/phoenix/schema/MetaDataClient.java | 192 +++++++- .../schema/tool/SchemaExtractionProcessor.java | 5 + .../java/org/apache/phoenix/util/MetaDataUtil.java | 5 + .../org/apache/phoenix/end2end/VectorIndexIT.java | 525 +++++++++++++++++++++ .../phoenix/compile/VectorIndexCompilerTest.java | 424 +++++++++++++++++ .../phoenix/parse/VectorColumnParseTest.java | 131 +++++ .../apache/phoenix/schema/VectorIndexTypeTest.java | 20 +- 10 files changed, 1455 insertions(+), 11 deletions(-) diff --git a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g index 31d274cb4a..872493704a 100644 --- a/phoenix-core-client/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core-client/src/main/antlr3/PhoenixSQL.g @@ -510,6 +510,7 @@ oneStatement returns [BindableStatement ret] | s=create_schema_node | s=create_view_node | s=create_index_node + | s=create_vector_index_node | s=create_cdc_node | s=cursor_open_node | s=cursor_close_node @@ -664,6 +665,22 @@ create_index_node returns [CreateIndexStatement ret] } ; +// Parse CREATE VECTOR INDEX. VECTOR is not a reserved word. The rule matches it as a NAME token, +// so other statements can use VECTOR as an identifier. +create_vector_index_node returns [CreateIndexStatement ret] + : CREATE vec=NAME {"VECTOR".equalsIgnoreCase($vec.text)}? INDEX (IF NOT ex=EXISTS)? i=index_name ON t=from_table_name + (LPAREN ik=ik_constraint RPAREN) + (INCLUDE (LPAREN icrefs=column_names RPAREN))? + ( (WITH? (LPAREN p=fam_properties RPAREN | p=fam_properties) (async=ASYNC)?) + | (async=ASYNC (WITH? (LPAREN p=fam_properties RPAREN | p=fam_properties))?) + )? + { + ret = factory.createIndex(i, factory.namedTable(null,t), ik, icrefs, null, p, ex!=null, + IndexType.VECTOR_GLOBAL, async != null, getBindCount(), new HashMap<String, + UDFParseNode>(udfParseNodes), null); + } + ; + create_cdc_node returns [CreateCDCStatement ret] : CREATE CDC (IF NOT ex=EXISTS)? o=cdc_name ON t=from_table_name (INCLUDE LPAREN v=cdc_change_scopes RPAREN)? diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java index f6e3b6eded..18e0262490 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java @@ -41,6 +41,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; +import org.apache.phoenix.optimize.DistanceMetric; import org.apache.phoenix.parse.CreateIndexStatement; import org.apache.phoenix.parse.NamedTableNode; import org.apache.phoenix.parse.ParseNode; @@ -51,6 +52,8 @@ import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.MultiKeyValueTuple; import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PBoolean; @@ -249,6 +252,9 @@ public class CreateIndexCompiler { new StatementContext(statement, resolver, scan, new SequenceManager(statement)); verifyIndexWhere(create.getWhere(), context, create.getTable().getName()); ExpressionCompiler expressionCompiler = new ExpressionCompiler(context); + if (create.getIndexType() == IndexType.VECTOR_GLOBAL) { + verifyVectorIndex(create, context); + } List<ParseNode> splitNodes = create.getSplitNodes(); if (create.getIndexType() == IndexType.LOCAL) { if (!splitNodes.isEmpty()) { @@ -292,6 +298,74 @@ public class CreateIndexCompiler { }; } + /** + * Validates vector index constraints including base table type, indexed expression type and + * dimensionality, distance metric, and algorithm parameters. + */ + private void verifyVectorIndex(CreateIndexStatement create, StatementContext context) + throws SQLException { + PTable dataTable = context.getResolver().getTables().get(0).getTable(); + if (dataTable.getType() == PTableType.VIEW) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("Vector indexes are not supported on views").build().buildException(); + } + + List<Pair<ParseNode, SortOrder>> pairs = + create.getIndexConstraint().getParseNodeAndSortOrderList(); + if (pairs.size() != 1) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("A vector index must index exactly one vector expression. Found: " + + pairs.size() + " expressions") + .build().buildException(); + } + ParseNode node = pairs.get(0).getFirst(); + if (pairs.get(0).getSecond() == SortOrder.DESC) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("The vector expression of a vector index cannot be declared DESC").build() + .buildException(); + } + Expression vectorExpr = StatementNormalizer.normalize(node, context.getResolver()) + .accept(new ExpressionCompiler(context)); + if (!vectorExpr.getDataType().isVectorType()) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.VECTOR_INDEX_ON_NON_VECTOR_TYPE) + .setMessage( + "Expression '" + node + "' has type " + vectorExpr.getDataType().getSqlTypeName()) + .build().buildException(); + } + Integer dimension = vectorExpr.getMaxLength(); + if (dimension == null || dimension <= 0) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("The vector expression dimension must be known at compile time").build() + .buildException(); + } + + String metric = create.getVectorMetric(); + if (metric == null || DistanceMetric.fromString(metric) == null) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC) + .setMessage(String.valueOf(metric)).build().buildException(); + } + String algorithm = create.getVectorAlgorithm(); + if (!"IVF".equalsIgnoreCase(algorithm == null ? null : algorithm.trim())) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_VECTOR_INDEX_ALGORITHM) + .setMessage(String.valueOf(algorithm)).build().buildException(); + } + Integer lists = create.getVectorLists(); + Integer sampleSize = create.getVectorSampleSize(); + if (lists == null || sampleSize == null) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("IVF requires lists and sample_size").build().buildException(); + } + if (lists <= 0 || sampleSize < lists) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("lists=" + lists + ", sample_size=" + sampleSize).build().buildException(); + } + if (create.getIndexConsistency() != null && create.getIndexConsistency().isAsynchronous()) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("Vector indexes do not support CONSISTENCY=" + create.getIndexConsistency()) + .build().buildException(); + } + } + /** * Helper method to validate CREATE INDEX statements on SYSTEM tables. 1. Pass if scheme name not * provided, assumption is - it not a SYSTEM table. 2. Fail if SYSTEM_CATALOG_INDEXES_ENABLED not diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java index 8cc7685a31..993acaa4f5 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateIndexStatement.java @@ -18,8 +18,10 @@ package org.apache.phoenix.parse; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PTable.IndexType; @@ -40,6 +42,16 @@ public class CreateIndexStatement extends SingleTableStatement { private final Map<String, UDFParseNode> udfParseNodes; private final ParseNode where; private final IndexConsistency indexConsistency; + private final String vectorAlgorithm; + private final String vectorMetric; + private final Integer vectorLists; + private final Integer vectorSampleSize; + + /** Option names for the WITH clause of CREATE VECTOR INDEX. The name match ignores case. */ + public static final String VECTOR_ALGORITHM_OPTION = "ALGORITHM"; + public static final String VECTOR_METRIC_OPTION = "METRIC"; + public static final String VECTOR_LISTS_OPTION = "LISTS"; + public static final String VECTOR_SAMPLE_SIZE_OPTION = "SAMPLE_SIZE"; public CreateIndexStatement(NamedNode indexTableName, NamedTableNode dataTable, IndexKeyConstraint indexKeyConstraint, List<ColumnName> includeColumns, List<ParseNode> splits, @@ -69,6 +81,21 @@ public class CreateIndexStatement extends SingleTableStatement { this.udfParseNodes = udfParseNodes; this.where = where; this.indexConsistency = indexConsistency; + if (indexType == IndexType.VECTOR_GLOBAL) { + // Remove the vector options from the properties. The physical HBase table descriptor must + // not contain them. + this.vectorAlgorithm = + Objects.toString(removeVectorOption(this.props, VECTOR_ALGORITHM_OPTION), null); + this.vectorMetric = + Objects.toString(removeVectorOption(this.props, VECTOR_METRIC_OPTION), null); + this.vectorLists = toInteger(removeVectorOption(this.props, VECTOR_LISTS_OPTION)); + this.vectorSampleSize = toInteger(removeVectorOption(this.props, VECTOR_SAMPLE_SIZE_OPTION)); + } else { + this.vectorAlgorithm = null; + this.vectorMetric = null; + this.vectorLists = null; + this.vectorSampleSize = null; + } } public CreateIndexStatement(CreateIndexStatement createStmt, @@ -85,6 +112,10 @@ public class CreateIndexStatement extends SingleTableStatement { this.udfParseNodes = createStmt.getUdfParseNodes(); this.where = createStmt.where; this.indexConsistency = createStmt.getIndexConsistency(); + this.vectorAlgorithm = createStmt.getVectorAlgorithm(); + this.vectorMetric = createStmt.getVectorMetric(); + this.vectorLists = createStmt.getVectorLists(); + this.vectorSampleSize = createStmt.getVectorSampleSize(); } public static IndexConsistency @@ -103,6 +134,32 @@ public class CreateIndexStatement extends SingleTableStatement { return indexConsistency; } + private static Object removeVectorOption(ListMultimap<String, Pair<String, Object>> props, + String option) { + Object value = null; + Iterator<Pair<String, Object>> it = + props.get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY).iterator(); + while (it.hasNext()) { + Pair<String, Object> prop = it.next(); + if (option.equalsIgnoreCase(prop.getFirst())) { + value = prop.getSecond(); + it.remove(); + } + } + return value; + } + + private static Integer toInteger(Object value) { + if (value instanceof Number) { + return ((Number) value).intValue(); + } + try { + return value == null ? null : Integer.valueOf(value.toString().trim()); + } catch (NumberFormatException e) { + return null; + } + } + public IndexKeyConstraint getIndexConstraint() { return indexKeyConstraint; } @@ -146,4 +203,20 @@ public class CreateIndexStatement extends SingleTableStatement { public IndexConsistency getIndexConsistency() { return indexConsistency; } + + public String getVectorAlgorithm() { + return vectorAlgorithm; + } + + public String getVectorMetric() { + return vectorMetric; + } + + public Integer getVectorLists() { + return vectorLists; + } + + public Integer getVectorSampleSize() { + return vectorSampleSize; + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 1d7ff320f0..0f3368a234 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -109,6 +109,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DIMENSION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -225,6 +231,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.monitoring.TableMetricsManager; +import org.apache.phoenix.optimize.DistanceMetric; import org.apache.phoenix.parse.AddColumnStatement; import org.apache.phoenix.parse.AlterIndexStatement; import org.apache.phoenix.parse.ChangePermsStatement; @@ -284,6 +291,7 @@ import org.apache.phoenix.schema.types.IndexConsistency; import org.apache.phoenix.schema.types.PBson; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PDate; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PTimestamp; import org.apache.phoenix.schema.types.PUnsignedLong; @@ -349,9 +357,11 @@ public class MetaDataClient { + IMMUTABLE_STORAGE_SCHEME + "," + ENCODING_SCHEME + "," + USE_STATS_FOR_PARALLELIZATION + "," + VIEW_INDEX_ID_DATA_TYPE + "," + CHANGE_DETECTION_ENABLED + "," + PHYSICAL_TABLE_NAME + "," + SCHEMA_VERSION + "," + STREAMING_TOPIC_NAME + "," + INDEX_WHERE + "," + CDC_INCLUDE_TABLE - + "," + TTL + "," + ROW_KEY_MATCHER + "," + IS_STRICT_TTL + "," + INDEX_CONSISTENCY + + "," + TTL + "," + ROW_KEY_MATCHER + "," + IS_STRICT_TTL + "," + INDEX_CONSISTENCY + "," + + VECTOR_INDEX_ALGORITHM + "," + VECTOR_DISTANCE_METRIC + "," + VECTOR_DIMENSION + "," + + VECTOR_IVF_LISTS + "," + VECTOR_IVF_SAMPLE_SIZE + "," + VECTOR_CENTROID_GENERATION + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " - + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)"; @@ -1716,6 +1726,18 @@ public class MetaDataClient { col.getName().getString(), col.isRowTimestamp())); } + boolean isVectorIndex = statement.getIndexType() == IndexType.VECTOR_GLOBAL; + if (isVectorIndex) { + ColumnName centroidColName = + ColumnName.caseSensitiveColumnName(MetaDataUtil.VECTOR_CENTROID_ID_COLUMN_NAME); + allPkColumns + .add(new ColumnDefInPkConstraint(centroidColName, SortOrder.getDefault(), false)); + columnDefs.add(FACTORY.columnDef(centroidColName, PInteger.INSTANCE.getSqlTypeName(), false, + null, null, false, SortOrder.getDefault(), null, false)); + } + List<ColumnDef> vectorColDefs = isVectorIndex ? new ArrayList<ColumnDef>() : null; + Integer vectorDimension = null; + PhoenixStatement phoenixStatment = new PhoenixStatement(connection); StatementContext context = new StatementContext(phoenixStatment, resolver); IndexExpressionCompiler expressionIndexCompiler = new IndexExpressionCompiler(context); @@ -1784,12 +1806,35 @@ public class MetaDataClient { colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(null, name)); } indexedColumnNames.add(colName); - PDataType dataType = - IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); - allPkColumns.add(new ColumnDefInPkConstraint(colName, pair.getSecond(), isRowTimestamp)); - columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), - expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, - pair.getSecond(), expressionStr, isRowTimestamp)); + if (isVectorIndex) { + String defaultFamily = dataTable.getDefaultFamilyName() != null + ? dataTable.getDefaultFamilyName().getString() + : QueryConstants.DEFAULT_COLUMN_FAMILY; + ColumnName vectorColDefName; + if (colRef != null) { + PColumn column = colRef.getColumn(); + String columnFamilyName = + column.getFamilyName() != null ? column.getFamilyName().getString() : defaultFamily; + vectorColDefName = ColumnName.caseSensitiveColumnName(columnFamilyName, + IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); + } else { + String name = expressionStr.replaceAll("\"", "'"); + vectorColDefName = ColumnName.caseSensitiveColumnName(defaultFamily, + IndexUtil.getIndexColumnName(null, name)); + } + vectorDimension = expression.getMaxLength(); + vectorColDefs + .add(FACTORY.columnDef(vectorColDefName, expression.getDataType().getSqlTypeName(), + expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, + pair.getSecond(), expressionStr, isRowTimestamp)); + } else { + PDataType dataType = + IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); + allPkColumns.add(new ColumnDefInPkConstraint(colName, pair.getSecond(), isRowTimestamp)); + columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), + expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, + pair.getSecond(), expressionStr, isRowTimestamp)); + } } // Next all the PK columns from the data table that aren't indexed @@ -1812,6 +1857,10 @@ public class MetaDataClient { } } + if (isVectorIndex && vectorColDefs != null) { + columnDefs.addAll(vectorColDefs); + } + // Last all the included columns (minus any PK columns) for (ColumnName colName : includedColumns) { PColumn col = resolver.resolveColumn(null, colName.getFamilyName(), colName.getColumnName()) @@ -1910,6 +1959,16 @@ public class MetaDataClient { tableProps.put(MetaDataUtil.DATA_TABLE_NAME_PROP_NAME, dataTable.getPhysicalName().getString()); + if (isVectorIndex) { + // Send the normalized vector index metadata to createTableInternal in tableProps, which + // writes it to SYSTEM.CATALOG. + tableProps.put(VECTOR_INDEX_ALGORITHM, statement.getVectorAlgorithm().toUpperCase()); + tableProps.put(VECTOR_DISTANCE_METRIC, + DistanceMetric.fromString(statement.getVectorMetric()).name()); + tableProps.put(VECTOR_DIMENSION, vectorDimension); + tableProps.put(VECTOR_IVF_LISTS, statement.getVectorLists()); + tableProps.put(VECTOR_IVF_SAMPLE_SIZE, statement.getVectorSampleSize()); + } CreateTableStatement tableStatement = FACTORY.createTable(indexTableName, statement.getProps(), columnDefs, pk, statement.getSplitNodes(), PTableType.INDEX, statement.ifNotExists(), null, statement.getWhere(), statement.getBindCount(), null); @@ -1945,6 +2004,15 @@ public class MetaDataClient { return new MutationState(0, 0, connection); } + // Vector indexes remain in BUILDING state until centroids are initialized and assigned. + if (table.isVectorIndex()) { + if (ValidateLastDDLTimestampUtil.getValidateLastDdlTimestampEnabled(connection)) { + connection.removeTable(connection.getTenantId(), dataTable.getName().getString(), null, + dataTable.getTimeStamp()); + } + return new MutationState(0, 0, connection); + } + // If our connection is at a fixed point-in-time, we need to open a new // connection so that our new index table is visible. if (connection.getSCN() != null) { @@ -3620,6 +3688,9 @@ public class MetaDataClient { defaultCreateState = PIndexState.BUILDING; } } + if (indexType == IndexType.VECTOR_GLOBAL) { + defaultCreateState = PIndexState.BUILDING; + } PIndexState indexState = parent == null || (tableType == PTableType.VIEW || tableType == PTableType.CDC) ? null @@ -3627,6 +3698,22 @@ public class MetaDataClient { if (indexState == null && tableProps.containsKey(INDEX_STATE)) { indexState = PIndexState.fromSerializedValue(tableProps.get(INDEX_STATE).toString()); } + // Remove the vector metadata from tableProps. SYSTEM.CATALOG keeps these values, and the + // HBase table descriptor must not contain them. + String vectorIndexAlgorithm = null; + String vectorDistanceMetric = null; + Integer vectorDimension = null; + Integer vectorIvfLists = null; + Integer vectorIvfSampleSize = null; + Long vectorCentroidGeneration = null; + if (indexType == IndexType.VECTOR_GLOBAL) { + vectorIndexAlgorithm = (String) tableProps.remove(VECTOR_INDEX_ALGORITHM); + vectorDistanceMetric = (String) tableProps.remove(VECTOR_DISTANCE_METRIC); + vectorDimension = (Integer) tableProps.remove(VECTOR_DIMENSION); + vectorIvfLists = (Integer) tableProps.remove(VECTOR_IVF_LISTS); + vectorIvfSampleSize = (Integer) tableProps.remove(VECTOR_IVF_SAMPLE_SIZE); + vectorCentroidGeneration = (Long) tableProps.remove(VECTOR_CENTROID_GENERATION); + } PreparedStatement tableUpsert = connection.prepareStatement(CREATE_TABLE); tableUpsert.setString(1, tenantIdStr); tableUpsert.setString(2, schemaName); @@ -3765,6 +3852,42 @@ public class MetaDataClient { tableUpsert.setNull(39, Types.CHAR); } + if (vectorIndexAlgorithm == null) { + tableUpsert.setNull(40, Types.VARCHAR); + } else { + tableUpsert.setString(40, vectorIndexAlgorithm); + } + + if (vectorDistanceMetric == null) { + tableUpsert.setNull(41, Types.VARCHAR); + } else { + tableUpsert.setString(41, vectorDistanceMetric); + } + + if (vectorDimension == null) { + tableUpsert.setNull(42, Types.INTEGER); + } else { + tableUpsert.setInt(42, vectorDimension); + } + + if (vectorIvfLists == null) { + tableUpsert.setNull(43, Types.INTEGER); + } else { + tableUpsert.setInt(43, vectorIvfLists); + } + + if (vectorIvfSampleSize == null) { + tableUpsert.setNull(44, Types.INTEGER); + } else { + tableUpsert.setInt(44, vectorIvfSampleSize); + } + + if (vectorCentroidGeneration == null) { + tableUpsert.setNull(45, Types.BIGINT); + } else { + tableUpsert.setLong(45, vectorCentroidGeneration); + } + tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -3902,7 +4025,10 @@ public class MetaDataClient { .setTTL(ttl == null || ttl.equals(TTL_EXPRESSION_NOT_DEFINED) ? TTLExpressionFactory.create(ttlFromHierarchy) : TTLExpressionFactory.create(ttl)) - .setRowKeyMatcher(rowKeyMatcher).build(); + .setRowKeyMatcher(rowKeyMatcher).setVectorIndexAlgorithm(vectorIndexAlgorithm) + .setVectorDistanceMetric(vectorDistanceMetric).setVectorDimension(vectorDimension) + .setVectorIvfLists(vectorIvfLists).setVectorIvfSampleSize(vectorIvfSampleSize) + .setVectorCentroidGeneration(vectorCentroidGeneration).build(); result = new MetaDataMutationResult(code, result.getMutationTime(), table, true); addTableToCache(result, false); return table; @@ -4404,6 +4530,13 @@ public class MetaDataClient { parentTableName, result.getMutationTime()); if (table != null) { + if (tableType == PTableType.INDEX) { + deleteVectorCentroids(table); + } else if (tableType == PTableType.TABLE) { + for (PTable index : table.getIndexes()) { + deleteVectorCentroids(index); + } + } boolean dropMetaData = connection.getQueryServices().getProps() .getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); long ts = (scn == null ? result.getMutationTime() : scn); @@ -4492,6 +4625,42 @@ public class MetaDataClient { } } + /** + * Deletes SYSTEM.VECTOR_CENTROID entries for a dropped vector index using an isolated connection + * to preserve caller transaction state. + */ + private void deleteVectorCentroids(PTable index) { + if (!index.isVectorIndex()) { + return; + } + String indexName = index.getName().getString(); + try { + Properties props = new Properties(); + Properties clientInfo = connection.getClientInfo(); + for (String key : clientInfo.stringPropertyNames()) { + if ( + !PhoenixRuntime.TENANT_ID_ATTRIB.equals(key) + && !PhoenixRuntime.CURRENT_SCN_ATTRIB.equals(key) + ) { + props.setProperty(key, clientInfo.getProperty(key)); + } + } + try ( + PhoenixConnection conn = + new PhoenixConnection(connection, connection.getQueryServices(), props); + PreparedStatement ps = + conn.prepareStatement("DELETE FROM " + PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_NAME + + " WHERE " + PhoenixDatabaseMetaData.INDEX_NAME + " = ?")) { + conn.setAutoCommit(true); + ps.setString(1, indexName); + ps.executeUpdate(); + } + } catch (SQLException e) { + LOGGER.warn("Could not delete the centroids of dropped vector index {}; its " + + PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_NAME + " rows remain.", indexName, e); + } + } + private MutationCode processMutationResult(String schemaName, String tableName, MetaDataMutationResult result) throws SQLException { final MutationCode mutationCode = result.getMutationCode(); @@ -5991,6 +6160,11 @@ public class MetaDataClient { IndexConsistency newIndexConsistency = statement.getIndexConsistency(); if (newIndexConsistency != null) { + if (newIndexConsistency.isAsynchronous() && table.isVectorIndex()) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("Vector indexes do not support CONSISTENCY=" + newIndexConsistency) + .setSchemaName(schemaName).setTableName(indexName).build().buildException(); + } try (PreparedStatement consistencyUpsert = connection.prepareStatement(UPDATE_INDEX_CONSISTENCY)) { consistencyUpsert.setString(1, tenantId); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java index d79e5010f1..0468b284db 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java @@ -27,6 +27,7 @@ import static org.apache.phoenix.util.MetaDataUtil.convertForeverAndNoneTTLValue import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -108,6 +109,10 @@ public class SchemaExtractionProcessor implements SchemaProcessor { } protected String extractCreateIndexDDL(PTable indexPTable) throws SQLException, IOException { + if (indexPTable.isVectorIndex()) { + throw new SQLFeatureNotSupportedException( + "Schema extraction does not support vector indexes: " + indexPTable.getName()); + } String quotedIndexTableName = SchemaUtil.getFullTableNameWithQuotes(null, indexPTable.getTableName().getString()); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 9249a8ad68..469125a48b 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -102,6 +102,11 @@ public class MetaDataUtil { public static final byte[] VIEW_INDEX_SEQUENCE_PREFIX_BYTES = Bytes.toBytes(VIEW_INDEX_SEQUENCE_PREFIX); public static final String VIEW_INDEX_ID_COLUMN_NAME = "_INDEX_ID"; + /** + * Row key column of an IVF vector index that holds the centroid ID of the posting list for the + * row. It is the first key column after the salt byte and the tenant ID, if they exist. + */ + public static final String VECTOR_CENTROID_ID_COLUMN_NAME = "_CENTROID_ID"; public static final String PARENT_TABLE_KEY = "PARENT_TABLE"; public static final String IS_VIEW_INDEX_TABLE_PROP_NAME = "IS_VIEW_INDEX_TABLE"; public static final byte[] IS_VIEW_INDEX_TABLE_PROP_BYTES = diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java index 074a8b1753..b8d5e97f47 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java @@ -17,13 +17,18 @@ */ package org.apache.phoenix.end2end; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_VECTOR; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GENERATION_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES; @@ -33,22 +38,29 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGOR import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE_BYTES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; @@ -63,20 +75,31 @@ import org.apache.phoenix.hbase.index.util.VersionUtil; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.protobuf.ProtobufUtil; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.types.PBson; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PDouble; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.types.PVectorFloat; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.ClientUtil; import org.apache.phoenix.util.Closeables; import org.apache.phoenix.util.IndexUtil; +import org.apache.phoenix.util.MetaDataUtil; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.StringUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -400,4 +423,506 @@ public class VectorIndexIT extends ParallelStatsDisabledIT { Closeables.closeQuietly(ht); } } + + /** + * Verifies that CREATE VECTOR INDEX writes the vector metadata to the catalog and creates the + * expected index table layout. + */ + @Test + public void testCreateVectorIndexTableAndMetadata() throws Exception { + String tableName = "T_VEC_DDL_" + generateUniqueName(); + String indexName = "IDX_VEC_DDL_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 16, sample_size = 500)"); + } + + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery( + "SELECT INDEX_TYPE, INDEX_STATE, VECTOR_INDEX_ALGORITHM, VECTOR_DISTANCE_METRIC, " + + "VECTOR_DIMENSION, VECTOR_IVF_LISTS, VECTOR_IVF_SAMPLE_SIZE, VECTOR_CENTROID_GENERATION " + + "FROM SYSTEM.CATALOG WHERE TABLE_NAME = '" + indexName + + "' AND TABLE_SCHEM IS NULL AND TENANT_ID IS NULL")) { + assertTrue("Expected row in SYSTEM.CATALOG for index table", rs.next()); + assertEquals(IndexType.VECTOR_GLOBAL.getSerializedValue(), rs.getByte("INDEX_TYPE")); + assertEquals(PIndexState.BUILDING.getSerializedValue(), rs.getString("INDEX_STATE")); + assertEquals("IVF", rs.getString("VECTOR_INDEX_ALGORITHM")); + assertEquals("L2", rs.getString("VECTOR_DISTANCE_METRIC")); + assertEquals(128, rs.getInt("VECTOR_DIMENSION")); + assertEquals(16, rs.getInt("VECTOR_IVF_LISTS")); + assertEquals(500, rs.getInt("VECTOR_IVF_SAMPLE_SIZE")); + long gen = rs.getLong("VECTOR_CENTROID_GENERATION"); + assertTrue("Centroid generation should be 0 or null", gen == 0 || rs.wasNull()); + } + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull("Index PTable must not be null", indexTable); + assertTrue("Table must be recognized as vector index", indexTable.isVectorIndex()); + assertEquals(IndexType.VECTOR_GLOBAL, indexTable.getIndexType()); + assertEquals(PIndexState.BUILDING, indexTable.getIndexState()); + assertEquals("IVF", indexTable.getVectorIndexAlgorithm()); + assertEquals("L2", indexTable.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(128), indexTable.getVectorDimension()); + assertEquals(Integer.valueOf(16), indexTable.getVectorIvfLists()); + assertEquals(Integer.valueOf(500), indexTable.getVectorIvfSampleSize()); + + // The row key starts with the centroid ID. The data table primary key columns follow it. + List<PColumn> pkColumns = indexTable.getPKColumns(); + assertEquals("Row key must have 2 PK columns", 2, pkColumns.size()); + PColumn pk0 = pkColumns.get(0); + assertEquals(MetaDataUtil.VECTOR_CENTROID_ID_COLUMN_NAME, pk0.getName().getString()); + assertEquals(PInteger.INSTANCE, pk0.getDataType()); + assertFalse("Centroid ID column must not be nullable", pk0.isNullable()); + + PColumn pk1 = pkColumns.get(1); + assertEquals(":ID", pk1.getName().getString()); + assertEquals(PVarchar.INSTANCE, pk1.getDataType()); + + // The indexed vector is a column in a column family. It is not part of the row key. + PColumn vectorCol = indexTable.getColumnForColumnName("0:V"); + assertNotNull("Vector column 0:V must exist in index table", vectorCol); + assertEquals(PVectorFloat.INSTANCE, vectorCol.getDataType()); + assertEquals(Integer.valueOf(128), vectorCol.getMaxLength()); + assertNotNull("Vector column must belong to a column family", vectorCol.getFamilyName()); + + // The vector options are catalog metadata only. The HBase table descriptor must not contain + // them. + try (Admin admin = pconn.getQueryServices().getAdmin()) { + TableDescriptor td = admin.getDescriptor( + org.apache.hadoop.hbase.TableName.valueOf(indexTable.getPhysicalName().getString())); + for (String key : new String[] { "METRIC", "ALGORITHM", "LISTS", "SAMPLE_SIZE", + PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM, + PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC, PhoenixDatabaseMetaData.VECTOR_DIMENSION, + PhoenixDatabaseMetaData.VECTOR_IVF_LISTS, PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE, + PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION }) { + assertNull("Index descriptor must not carry " + key, td.getValue(key)); + } + } + } + } + + /** + * Verifies that a data column named CENTROID_ID does not collide with the centroid ID column at + * the start of the index row key. + */ + @Test + public void testCentroidColumnDoesNotCollideWithDataColumn() throws Exception { + String tableName = "T_CENTROID_PK_" + generateUniqueName(); + String indexName = "IDX_CENTROID_PK_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE " + tableName + + " (CENTROID_ID INTEGER NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 4))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 2, sample_size = 10)"); + } + PTable indexTable = conn.unwrap(PhoenixConnection.class).getTableNoCache(indexName); + List<PColumn> pkColumns = indexTable.getPKColumns(); + assertEquals(2, pkColumns.size()); + assertEquals(MetaDataUtil.VECTOR_CENTROID_ID_COLUMN_NAME, + pkColumns.get(0).getName().getString()); + assertEquals(":CENTROID_ID", pkColumns.get(1).getName().getString()); + } + } + + /** Verifies that the INCLUDE columns of a vector index go into the default column family. */ + @Test + public void testCreateVectorIndexWithIncludeColumns() throws Exception { + String tableName = "T_VEC_INC_" + generateUniqueName(); + String indexName = "IDX_VEC_INC_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE " + tableName + + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 64), LABEL VARCHAR, SCORE DOUBLE)"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "INCLUDE (LABEL, SCORE) " + + "WITH (algorithm = 'IVF', metric = 'COSINE', lists = 8, sample_size = 250)"); + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull(indexTable); + assertTrue(indexTable.isVectorIndex()); + assertEquals(IndexType.VECTOR_GLOBAL, indexTable.getIndexType()); + assertEquals(PIndexState.BUILDING, indexTable.getIndexState()); + + List<PColumn> pkColumns = indexTable.getPKColumns(); + assertEquals(2, pkColumns.size()); + assertEquals(MetaDataUtil.VECTOR_CENTROID_ID_COLUMN_NAME, + pkColumns.get(0).getName().getString()); + assertEquals(":ID", pkColumns.get(1).getName().getString()); + + PColumn vectorCol = indexTable.getColumnForColumnName("0:V"); + assertNotNull("Vector column 0:V must exist", vectorCol); + assertEquals(PVectorFloat.INSTANCE, vectorCol.getDataType()); + + PColumn labelCol = indexTable.getColumnForColumnName("0:LABEL"); + assertNotNull("Covered column 0:LABEL must exist", labelCol); + assertEquals(PVarchar.INSTANCE, labelCol.getDataType()); + + PColumn scoreCol = indexTable.getColumnForColumnName("0:SCORE"); + assertNotNull("Covered column 0:SCORE must exist", scoreCol); + assertEquals(PDouble.INSTANCE, scoreCol.getDataType()); + } + } + + /** Verifies index creation and catalog metadata for a double-precision vector column. */ + @Test + public void testCreateVectorIndexWithDoubleVectors() throws Exception { + String tableName = "T_VEC_DBL_" + generateUniqueName(); + String indexName = "IDX_VEC_DBL_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(DOUBLE, 64))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'COSINE', " + + "dimension = 64, lists = 32, sample_size = 1000)"); + } + + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery( + "SELECT INDEX_TYPE, INDEX_STATE, VECTOR_INDEX_ALGORITHM, VECTOR_DISTANCE_METRIC, " + + "VECTOR_DIMENSION, VECTOR_IVF_LISTS, VECTOR_IVF_SAMPLE_SIZE " + + "FROM SYSTEM.CATALOG WHERE TABLE_NAME = '" + indexName + + "' AND TABLE_SCHEM IS NULL AND TENANT_ID IS NULL")) { + assertTrue("Expected row in SYSTEM.CATALOG for vector index", rs.next()); + assertEquals(IndexType.VECTOR_GLOBAL.getSerializedValue(), rs.getByte("INDEX_TYPE")); + assertEquals(PIndexState.BUILDING.getSerializedValue(), rs.getString("INDEX_STATE")); + assertEquals("IVF", rs.getString("VECTOR_INDEX_ALGORITHM")); + assertEquals("COSINE", rs.getString("VECTOR_DISTANCE_METRIC")); + assertEquals(64, rs.getInt("VECTOR_DIMENSION")); + assertEquals(32, rs.getInt("VECTOR_IVF_LISTS")); + assertEquals(1000, rs.getInt("VECTOR_IVF_SAMPLE_SIZE")); + } + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull(indexTable); + assertTrue(indexTable.isVectorIndex()); + assertEquals(IndexType.VECTOR_GLOBAL, indexTable.getIndexType()); + assertEquals(PIndexState.BUILDING, indexTable.getIndexState()); + + List<PColumn> pkColumns = indexTable.getPKColumns(); + assertEquals(2, pkColumns.size()); + assertEquals(MetaDataUtil.VECTOR_CENTROID_ID_COLUMN_NAME, + pkColumns.get(0).getName().getString()); + assertEquals(":ID", pkColumns.get(1).getName().getString()); + } + } + + /** + * Verifies that a new vector index starts in the BUILDING state and gets its dimension from the + * data column. + */ + @Test + public void testVectorIndexStateIsBuildingAndInfersDimension() throws Exception { + String tableName = "T_VEC_INFER_" + generateUniqueName(); + String indexName = "IDX_VEC_INFER_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 32))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 4, sample_size = 100)"); + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull(indexTable); + assertEquals(PIndexState.BUILDING, indexTable.getIndexState()); + assertEquals(Integer.valueOf(32), indexTable.getVectorDimension()); + assertEquals(Integer.valueOf(4), indexTable.getVectorIvfLists()); + assertEquals(Integer.valueOf(100), indexTable.getVectorIvfSampleSize()); + } + } + + private Connection getDropMetadataConnection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.setProperty(QueryServices.DROP_METADATA_ATTRIB, Boolean.toString(true)); + props.setProperty(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, StringUtil.EMPTY_STRING); + String url = QueryUtil.getConnectionUrl(props, config, generateUniqueName()); + return DriverManager.getConnection(url, props); + } + + /** + * Verifies that DROP INDEX removes the centroids, the SYSTEM.CATALOG rows, and the physical table + * of a vector index. + */ + @Test + public void testDropVectorIndex() throws Exception { + String tableName = "T_DROP_VEC_" + generateUniqueName(); + String indexName = "IDX_DROP_VEC_" + generateUniqueName(); + + try (Connection conn = getDropMetadataConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 16, sample_size = 500)"); + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull(indexTable); + String fullIndexName = indexTable.getName().getString(); + String physicalName = indexTable.getPhysicalName().getString(); + org.apache.hadoop.hbase.TableName hbaseTableName = + org.apache.hadoop.hbase.TableName.valueOf(physicalName); + + String upsertCentroidSql = + "UPSERT INTO " + SYSTEM_VECTOR_CENTROID_NAME + " (" + INDEX_NAME + ", " + CENTROID_ID + ", " + + CENTROID_VECTOR + ", " + GENERATION_ID + ") VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(upsertCentroidSql)) { + for (int i = 0; i < 4; i++) { + ps.setString(1, fullIndexName); + ps.setInt(2, i); + ps.setBytes(3, new byte[] { (byte) i, 1, 2, 3 }); + ps.setLong(4, 1L); + ps.executeUpdate(); + } + } + conn.commit(); + + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, fullIndexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals(4, rs.getInt(1)); + } + } + + try (Admin admin = pconn.getQueryServices().getAdmin()) { + assertTrue("HBase physical table should exist before drop", + admin.tableExists(hbaseTableName)); + } + + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP INDEX " + indexName + " ON " + tableName); + } + + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, fullIndexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals("Centroids must be removed on drop index", 0, rs.getInt(1)); + } + } + + try (Admin admin = pconn.getQueryServices().getAdmin()) { + assertFalse("HBase physical table should be deleted on drop index", + admin.tableExists(hbaseTableName)); + } + + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1 FROM SYSTEM.CATALOG WHERE TABLE_NAME = '" + + indexName + "' AND TABLE_SCHEM IS NULL AND TENANT_ID IS NULL")) { + assertFalse("Index row in SYSTEM.CATALOG must be deleted", rs.next()); + } + + try { + pconn.getTableNoCache(indexName); + fail("Expected TableNotFoundException after drop"); + } catch (TableNotFoundException expected) { + } + } + } + + /** + * Verifies that DROP TABLE also removes the centroids and the physical table of each vector index + * on the data table. + */ + @Test + public void testDropTableCascadesToVectorIndexCentroids() throws Exception { + String tableName = "T_CASCADE_VEC_" + generateUniqueName(); + String indexName = "IDX_CASCADE_VEC_" + generateUniqueName(); + + try (Connection conn = getDropMetadataConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 64))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + tableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 8, sample_size = 250)"); + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(indexName); + assertNotNull(indexTable); + String fullIndexName = indexTable.getName().getString(); + String physicalName = indexTable.getPhysicalName().getString(); + org.apache.hadoop.hbase.TableName hbaseTableName = + org.apache.hadoop.hbase.TableName.valueOf(physicalName); + + String upsertCentroidSql = + "UPSERT INTO " + SYSTEM_VECTOR_CENTROID_NAME + " (" + INDEX_NAME + ", " + CENTROID_ID + ", " + + CENTROID_VECTOR + ", " + GENERATION_ID + ") VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(upsertCentroidSql)) { + for (int i = 0; i < 3; i++) { + ps.setString(1, fullIndexName); + ps.setInt(2, i); + ps.setBytes(3, new byte[] { (byte) i, 4, 5, 6 }); + ps.setLong(4, 1L); + ps.executeUpdate(); + } + } + conn.commit(); + + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP TABLE " + tableName); + } + + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, fullIndexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals("Centroids must be removed on cascade drop table", 0, rs.getInt(1)); + } + } + + try (Admin admin = pconn.getQueryServices().getAdmin()) { + assertFalse("HBase physical table for index should be deleted", + admin.tableExists(hbaseTableName)); + } + } + } + + /** Verifies that DROP INDEX removes the centroids of a vector index in a schema. */ + @Test + public void testDropVectorIndexWithSchema() throws Exception { + String schemaName = "S_" + generateUniqueName(); + String tableName = "T_DROP_VEC_" + generateUniqueName(); + String indexName = "IDX_DROP_VEC_" + generateUniqueName(); + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); + String fullIndexName = SchemaUtil.getTableName(schemaName, indexName); + + try (Connection conn = getDropMetadataConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE " + fullTableName + + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + stmt.execute("CREATE VECTOR INDEX " + indexName + " ON " + fullTableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 16, sample_size = 500)"); + } + + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable indexTable = pconn.getTableNoCache(fullIndexName); + assertNotNull(indexTable); + assertEquals(fullIndexName, indexTable.getName().getString()); + + String upsertCentroidSql = + "UPSERT INTO " + SYSTEM_VECTOR_CENTROID_NAME + " (" + INDEX_NAME + ", " + CENTROID_ID + ", " + + CENTROID_VECTOR + ", " + GENERATION_ID + ") VALUES (?, ?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(upsertCentroidSql)) { + for (int i = 0; i < 3; i++) { + ps.setString(1, fullIndexName); + ps.setInt(2, i); + ps.setBytes(3, new byte[] { (byte) i, 1, 2, 3 }); + ps.setLong(4, 1L); + ps.executeUpdate(); + } + } + conn.commit(); + + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, fullIndexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals(3, rs.getInt(1)); + } + } + + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP INDEX " + indexName + " ON " + fullTableName); + } + + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, fullIndexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals("Centroids must be removed on drop index with schema", 0, rs.getInt(1)); + } + } + } + } + + /** Verifies that DROP INDEX removes the centroids of a vector index with a quoted name. */ + @Test + public void testDropQuotedVectorIndexRemovesCentroids() throws Exception { + String schemaName = "s" + generateUniqueName().toLowerCase(); + String tableName = "t" + generateUniqueName().toLowerCase(); + String indexName = "idx" + generateUniqueName().toLowerCase(); + String fullTableName = "\"" + schemaName + "\".\"" + tableName + "\""; + + try (Connection conn = getDropMetadataConnection()) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE " + fullTableName + + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 4))"); + stmt.execute("CREATE VECTOR INDEX \"" + indexName + "\" ON " + fullTableName + " (V) " + + "WITH (algorithm = 'IVF', metric = 'L2', lists = 2, sample_size = 10)"); + } + PTable indexTable = + conn.unwrap(PhoenixConnection.class).getTableNoCache(schemaName + "." + indexName); + // The centroid rows use the catalog name of the index as the key. This name keeps the case + // of the quoted identifiers. + String key = indexTable.getName().getString(); + assertEquals(schemaName + "." + indexName, key); + try (PreparedStatement ps = conn.prepareStatement( + "UPSERT INTO " + SYSTEM_VECTOR_CENTROID_NAME + " (" + INDEX_NAME + ", " + CENTROID_ID + ", " + + CENTROID_VECTOR + ", " + GENERATION_ID + ") VALUES (?, ?, ?, ?)")) { + for (int i = 0; i < 2; i++) { + ps.setString(1, key); + ps.setInt(2, i); + ps.setBytes(3, new byte[] { (byte) i }); + ps.setLong(4, 1L); + ps.executeUpdate(); + } + } + conn.commit(); + + try (Statement stmt = conn.createStatement()) { + stmt.execute("DROP INDEX \"" + indexName + "\" ON " + fullTableName); + } + try (PreparedStatement ps = conn.prepareStatement( + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?")) { + ps.setString(1, key); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals(0, rs.getInt(1)); + } + } + } + } + + /** + * Verifies that DROP INDEX IF EXISTS removes an existing vector index, and completes without + * error after the index no longer exists. + */ + @Test + public void testDropVectorIndexIfExists() throws Exception { + String tableName = "T_IF_EXISTS_" + generateUniqueName(); + String indexName = "IDX_NON_EXISTENT_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 32))"); + stmt.execute("DROP INDEX IF EXISTS " + indexName + " ON " + tableName); + } + } + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java new file mode 100644 index 0000000000..ec5104c497 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/VectorIndexCompilerTest.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compile; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.Properties; +import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.query.BaseConnectionlessQueryTest; +import org.apache.phoenix.util.PropertiesUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests the compile-time validation of vector index DDL, mainly in {@link CreateIndexCompiler}. + */ +public class VectorIndexCompilerTest extends BaseConnectionlessQueryTest { + + @BeforeClass + public static void setupTables() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + conn.createStatement() + .execute("CREATE TABLE T_VEC (ID VARCHAR NOT NULL PRIMARY KEY, V1 VECTOR(FLOAT, 128), " + + "V2 VECTOR(FLOAT, 128), S VARCHAR)"); + } + } + + @Test + public void testInvalidDistanceMetricRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='HAMMING', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for unsupported metric HAMMING"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC.getErrorCode(), + e.getErrorCode()); + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC.getSQLState(), + e.getSQLState()); + } + } + } + + @Test + public void testMultipleVectorColumnsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1, V2) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for multiple vector columns in index key constraint"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testMultipleColumnsMixedRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1, S) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for multiple key expressions in vector index"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testMissingRequiredIvfListsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = + "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2', sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for missing lists IVF parameter"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testMissingRequiredIvfSampleSizeRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = + "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2', lists=16)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for missing sample_size IVF parameter"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testMissingBothRequiredIvfParamsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2')"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for missing required IVF parameters"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testInvalidIvfZeroListsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2', " + + "lists=0, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for lists <= 0"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testInvalidIvfSampleSizeLessThanListsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2', " + + "lists=32, sample_size=16)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for sample_size < lists"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testInvalidAlgorithmRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='UNKNOWN', metric='L2', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for unknown algorithm"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_INDEX_ALGORITHM.getErrorCode(), + e.getErrorCode()); + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_INDEX_ALGORITHM.getSQLState(), + e.getSQLState()); + } + } + } + + @Test + public void testNonVectorColumnRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (S) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for non-vector column S"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.VECTOR_INDEX_ON_NON_VECTOR_TYPE.getErrorCode(), + e.getErrorCode()); + assertEquals(SQLExceptionCode.VECTOR_INDEX_ON_NON_VECTOR_TYPE.getSQLState(), + e.getSQLState()); + } + } + } + + /** + * Positive control for the rejection tests. Each supported metric compiles to a CREATE INDEX plan + * on the data table. The option values use mixed case and extra whitespace. The DDL also has an + * explicit ASC key, a sample size equal to the list count, and STRONG consistency. + */ + @Test + public void testValidVectorIndexCompilation() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + for (String metric : new String[] { "l2", " Cosine ", "INNER_PRODUCT" }) { + String ddl = + "CREATE VECTOR INDEX idx ON T_VEC (V1 ASC) INCLUDE (S) WITH (algorithm=' ivf ', " + + "metric='" + metric + "', lists=16, sample_size=16, CONSISTENCY=STRONG)"; + MutationPlan plan = stmt.compileMutation(ddl); + assertEquals(ddl, "T_VEC", plan.getTargetRef().getTable().getTableName().getString()); + assertEquals(ddl, Collections.singletonList("CREATE INDEX"), + plan.getExplainPlan().getPlanSteps()); + } + } + } + + /** + * Index maintenance always writes the indexed vector in ascending byte order. A DESC vector key + * would describe index data that maintenance never writes, so the compiler must reject it. + */ + @Test + public void testDescVectorKeyRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON T_VEC (V1 DESC) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"; + SQLException rejection = null; + try { + stmt.compileMutation(ddl); + } catch (SQLException e) { + rejection = e; + } + assertNotNull("Expected compilation error for a DESC vector index key", rejection); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), + rejection.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), + rejection.getSQLState()); + } + } + + /** + * The eventually consistent index consumer does not maintain vector indexes. Thus CREATE and + * ALTER must both reject eventual consistency for a vector index. + */ + @Test + public void testEventualConsistencyRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + try { + stmt.compileMutation("CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', " + + "metric='L2', lists=16, sample_size=500, CONSISTENCY=EVENTUAL)"); + fail("Expected compilation error for an eventually consistent vector index"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + + conn.createStatement() + .execute("CREATE TABLE T_VEC_EC (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 4))"); + conn.createStatement().execute("CREATE VECTOR INDEX I_VEC_EC ON T_VEC_EC (V) WITH " + + "(algorithm='IVF', metric='L2', lists=2, sample_size=4) ASYNC"); + try { + conn.createStatement().execute("ALTER INDEX I_VEC_EC ON T_VEC_EC CONSISTENCY=EVENTUAL"); + fail("Expected ALTER INDEX to refuse eventual consistency for a vector index"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testDdlExecutionInvalidDistanceMetricRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + try (Statement stmt = conn.createStatement()) { + stmt + .execute("CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='HAMMING', " + + "lists=16, sample_size=500)"); + fail("Expected compilation error for unsupported metric HAMMING during execute()"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC.getErrorCode(), + e.getErrorCode()); + assertEquals(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC.getSQLState(), + e.getSQLState()); + } + } + } + + @Test + public void testDdlExecutionMultipleVectorColumnsRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + try (Statement stmt = conn.createStatement()) { + stmt + .execute("CREATE VECTOR INDEX idx ON T_VEC (V1, V2) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"); + fail("Expected compilation error for multiple vector columns during execute()"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testDdlExecutionMissingRequiredParametersRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE VECTOR INDEX idx ON T_VEC (V1) WITH (algorithm='IVF', metric='L2')"); + fail("Expected compilation error for missing required IVF parameters during execute()"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testVectorIndexOnViewRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + conn.createStatement().execute("CREATE VIEW V_VEC AS SELECT * FROM T_VEC WHERE S = 'a'"); + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + String ddl = "CREATE VECTOR INDEX idx ON V_VEC (V1) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"; + try { + stmt.compileMutation(ddl); + fail("Expected compilation error for vector index on a view"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testDdlExecutionVectorIndexOnViewRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE VECTOR INDEX idx ON V_VEC (V1) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"); + fail("Expected compilation error for vector index on a view during execute()"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getErrorCode(), e.getErrorCode()); + assertEquals(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS.getSQLState(), e.getSQLState()); + } + } + } + + @Test + public void testDdlExecutionNonVectorColumnRejection() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (PhoenixConnection conn = + DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class)) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE VECTOR INDEX idx ON T_VEC (S) WITH (algorithm='IVF', metric='L2', " + + "lists=16, sample_size=500)"); + fail("Expected compilation error for non-vector column during execute()"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.VECTOR_INDEX_ON_NON_VECTOR_TYPE.getErrorCode(), + e.getErrorCode()); + assertEquals(SQLExceptionCode.VECTOR_INDEX_ON_NON_VECTOR_TYPE.getSQLState(), + e.getSQLState()); + } + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/parse/VectorColumnParseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/parse/VectorColumnParseTest.java index 3eb8dc97f3..56cdb5c9c4 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/parse/VectorColumnParseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/parse/VectorColumnParseTest.java @@ -20,12 +20,15 @@ package org.apache.phoenix.parse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.SQLException; import java.util.List; import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.types.PVectorDouble; import org.apache.phoenix.schema.types.PVectorFloat; @@ -219,4 +222,132 @@ public class VectorColumnParseTest { assertEquals("VECTOR", dynCol.getColumnDefName().getColumnName()); assertEquals("VARCHAR", dynCol.getDataType().getSqlTypeName()); } + + @Test + public void testParseVectorIndexFullSyntax() throws Exception { + String ddl = + "CREATE VECTOR INDEX idx ON t (v) WITH (metric='COSINE', algorithm='IVF', lists=32, sample_size=1000)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertEquals("IDX", indexStmt.getIndexTableName().getTableName()); + assertEquals("T", indexStmt.getTable().getName().getTableName()); + assertEquals(PTable.IndexType.VECTOR_GLOBAL, indexStmt.getIndexType()); + assertFalse("Expected ifNotExists to be false", indexStmt.ifNotExists()); + assertFalse("Expected async to be false", indexStmt.isAsync()); + assertEquals(1, indexStmt.getIndexConstraint().getParseNodeAndSortOrderList().size()); + + assertEquals("COSINE", indexStmt.getVectorMetric()); + assertEquals("IVF", indexStmt.getVectorAlgorithm()); + assertEquals(Integer.valueOf(32), indexStmt.getVectorLists()); + assertEquals(Integer.valueOf(1000), indexStmt.getVectorSampleSize()); + // The statement must remove the vector options from the physical table properties. + assertTrue(indexStmt.getProps().get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY).isEmpty()); + } + + @Test + public void testParseVectorIndexIfNotExists() throws Exception { + String ddl = + "CREATE VECTOR INDEX IF NOT EXISTS idx ON t (v) WITH (algorithm='IVF', metric='L2', lists=16, sample_size=500)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertTrue("Expected ifNotExists to be true", indexStmt.ifNotExists()); + assertEquals(PTable.IndexType.VECTOR_GLOBAL, indexStmt.getIndexType()); + } + + @Test + public void testParseVectorIndexAsync() throws Exception { + String ddl = + "CREATE VECTOR INDEX idx ON t (v) WITH (algorithm='IVF', metric='L2', lists=16, sample_size=500) ASYNC"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertTrue("Expected async to be true", indexStmt.isAsync()); + assertEquals(PTable.IndexType.VECTOR_GLOBAL, indexStmt.getIndexType()); + } + + @Test + public void testParseVectorIndexIncludeClause() throws Exception { + String ddl = + "CREATE VECTOR INDEX idx ON t (v) INCLUDE (col1, col2) WITH (metric='COSINE', algorithm='IVF', lists=32, sample_size=1000)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + List<ColumnName> includeCols = indexStmt.getIncludeColumns(); + assertEquals(2, includeCols.size()); + assertEquals("COL1", includeCols.get(0).getColumnName()); + assertEquals("COL2", includeCols.get(1).getColumnName()); + } + + @Test + public void testParseVectorIndexFunctionExpression() throws Exception { + String ddl = + "CREATE VECTOR INDEX idx ON t (BSON_VECTOR_VALUE(profile, 'search.embedding', 768)) " + + "INCLUDE (author) WITH (metric='COSINE', algorithm='IVF', lists=1024, sample_size=50000)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertEquals(1, indexStmt.getIndexConstraint().getParseNodeAndSortOrderList().size()); + assertTrue(indexStmt.getIndexConstraint().getParseNodeAndSortOrderList().get(0) + .getFirst() instanceof FunctionParseNode); + FunctionParseNode fn = (FunctionParseNode) indexStmt.getIndexConstraint() + .getParseNodeAndSortOrderList().get(0).getFirst(); + assertEquals("BSON_VECTOR_VALUE", fn.getName()); + assertEquals(1, indexStmt.getIncludeColumns().size()); + assertEquals("AUTHOR", indexStmt.getIncludeColumns().get(0).getColumnName()); + } + + @Test + public void testVectorIndexAccessorValues() throws Exception { + String ddl = + "CREATE VECTOR INDEX idx ON t (v) WITH (algorithm='IVF', metric='COSINE', lists=32, sample_size=1000)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertEquals("IVF", indexStmt.getVectorAlgorithm()); + assertEquals("COSINE", indexStmt.getVectorMetric()); + assertEquals(Integer.valueOf(32), indexStmt.getVectorLists()); + assertEquals(Integer.valueOf(1000), indexStmt.getVectorSampleSize()); + } + + @Test + public void testVectorIndexKeywordIsContextual() throws Exception { + // VECTOR is a keyword only in CREATE VECTOR INDEX. Other statements can use it as a name. + CreateIndexStatement plain = + (CreateIndexStatement) new SQLParser("CREATE INDEX vector ON t (v)").parseStatement(); + assertEquals("VECTOR", plain.getIndexTableName().getTableName()); + assertEquals(PTable.IndexType.GLOBAL, plain.getIndexType()); + + CreateIndexStatement vector = (CreateIndexStatement) new SQLParser( + "create vector index vector on t (v) with (algorithm='IVF', metric='L2', lists=2, " + + "sample_size=10)").parseStatement(); + assertEquals("VECTOR", vector.getIndexTableName().getTableName()); + assertEquals(PTable.IndexType.VECTOR_GLOBAL, vector.getIndexType()); + + try { + new SQLParser("CREATE VECTORS INDEX idx ON t (v)").parseStatement(); + fail("Only VECTOR may appear between CREATE and INDEX"); + } catch (SQLException expected) { + } + } + + @Test + public void testNonVectorIndexAccessorValues() throws Exception { + String ddl = "CREATE INDEX idx ON t (v)"; + SQLParser parser = new SQLParser(ddl); + BindableStatement stmt = parser.parseStatement(); + assertTrue("Expected CreateIndexStatement", stmt instanceof CreateIndexStatement); + CreateIndexStatement indexStmt = (CreateIndexStatement) stmt; + assertNull(indexStmt.getVectorAlgorithm()); + assertNull(indexStmt.getVectorMetric()); + assertNull(indexStmt.getVectorLists()); + assertNull(indexStmt.getVectorSampleSize()); + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java index 79b4007cd3..79abe73d50 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java @@ -25,10 +25,13 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.sql.SQLFeatureNotSupportedException; import java.util.Collections; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.generated.PTableProtos; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.tool.SchemaExtractionProcessor; import org.junit.Test; /** @@ -209,13 +212,13 @@ public class VectorIndexTypeTest { PTable original = new PTableImpl.Builder().setType(PTableType.INDEX) .setName(PNameFactory.newName("IDX_VEC")).setTableName(PNameFactory.newName("IDX_VEC")) .setIndexType(IndexType.VECTOR_GLOBAL).setParentTableName(PNameFactory.newName("DATA_TBL")) - .setPhysicalNames(Collections.emptyList()).setVectorIndexAlgorithm("DISKANN") + .setPhysicalNames(Collections.emptyList()).setVectorIndexAlgorithm("IVF") .setVectorDistanceMetric("INNER_PRODUCT").setVectorDimension(512).setVectorIvfLists(128) .setVectorIvfSampleSize(4096).setVectorCentroidGeneration(777L).build(); PTable cloned = PTableImpl.builderFromExisting(original).build(); assertTrue(cloned.isVectorIndex()); - assertEquals("DISKANN", cloned.getVectorIndexAlgorithm()); + assertEquals("IVF", cloned.getVectorIndexAlgorithm()); assertEquals("INNER_PRODUCT", cloned.getVectorDistanceMetric()); assertEquals(Integer.valueOf(512), cloned.getVectorDimension()); assertEquals(Integer.valueOf(128), cloned.getVectorIvfLists()); @@ -269,4 +272,17 @@ public class VectorIndexTypeTest { assertNull("vectorIvfSampleSize must be null", deserialized.getVectorIvfSampleSize()); assertNull("vectorCentroidGeneration must be null", deserialized.getVectorCentroidGeneration()); } + + @Test + public void testSchemaExtractionRejectsVectorIndex() throws Exception { + PTable index = new PTableImpl.Builder().setType(PTableType.INDEX) + .setIndexType(IndexType.VECTOR_GLOBAL).setName(PNameFactory.newName("IDX_VEC")) + .setTableName(PNameFactory.newName("IDX_VEC")).setSchemaName(PNameFactory.newName("")) + .setParentTableName(PNameFactory.newName("DATA_TBL")).build(); + try { + new SchemaExtractionProcessor(null, new Configuration(), index, false).process(); + fail("Schema extraction cannot reproduce CREATE VECTOR INDEX and must say so"); + } catch (SQLFeatureNotSupportedException expected) { + } + } }
