This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch PHOENIX-7998 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit aef42f6d45448c71ea05c32a83ee0e2c1a2fbf70 Author: Andrew Purtell <[email protected]> AuthorDate: Fri Sep 18 08:47:18 2026 -0700 PHOENIX-7998 Phase 4: System catalog and metadata extensions Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --- .../coprocessorclient/MetaDataProtocol.java | 2 +- .../apache/phoenix/exception/SQLExceptionCode.java | 5 + .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 35 ++ .../phoenix/query/ConnectionQueryServicesImpl.java | 57 ++- .../query/ConnectionlessQueryServicesImpl.java | 8 + .../org/apache/phoenix/query/QueryConstants.java | 27 +- .../org/apache/phoenix/schema/DelegateTable.java | 30 ++ .../java/org/apache/phoenix/schema/PTable.java | 29 +- .../java/org/apache/phoenix/schema/PTableImpl.java | 133 ++++++- phoenix-core-client/src/main/protobuf/PTable.proto | 6 + .../phoenix/coprocessor/MetaDataEndpointImpl.java | 149 +++++++- .../MigrateSystemTablesToSystemNamespaceIT.java | 18 +- .../end2end/SkipSystemTablesExistenceCheckIT.java | 2 +- .../SystemTablesCreationOnConnectionIT.java | 4 +- .../phoenix/end2end/VectorCentroidTableIT.java | 221 +++++++++++ .../org/apache/phoenix/end2end/VectorIndexIT.java | 403 +++++++++++++++++++++ .../phoenix/end2end/VectorSystemCatalogIT.java | 129 +++++++ .../apache/phoenix/schema/VectorIndexTypeTest.java | 272 ++++++++++++++ 18 files changed, 1503 insertions(+), 27 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java index 08cac7cbe9..3052f69b38 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java @@ -92,7 +92,7 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_1_0 = MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_2_0 = MIN_TABLE_TIMESTAMP + 38; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 = MIN_TABLE_TIMESTAMP + 44; - public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 = MIN_TABLE_TIMESTAMP + 45; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 = MIN_TABLE_TIMESTAMP + 51; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* // constants public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index 3611dbf1df..0abafeefbd 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -121,6 +121,11 @@ public enum SQLExceptionCode { "Invalid index on table. Indexes on SYSTEM tables are not enabled."), CANNOT_INDEX_SYSTEM_TABLE(306, "23104", "Invalid index on table. SYSTEM Indexes can only be on SYSTEM.CATALOG table."), + VECTOR_INDEX_ON_NON_VECTOR_TYPE(307, "23105", + "Vector index can only be created on an expression of VECTOR type."), + UNSUPPORTED_VECTOR_INDEX_ALGORITHM(309, "23107", "Unsupported vector index algorithm: "), + UNSUPPORTED_VECTOR_DISTANCE_METRIC(310, "23108", "Unsupported vector distance metric: "), + INVALID_VECTOR_INDEX_PARAMS(311, "23109", "Invalid vector index parameters: "), /** * Invalid Cursor State (errorcode 04, sqlstate 24) */ diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index ed63485440..399fd9a88f 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -451,6 +451,25 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final String INDEX_WHERE = "INDEX_WHERE"; public static final byte[] INDEX_WHERE_BYTES = Bytes.toBytes(INDEX_WHERE); + public static final String VECTOR_INDEX_ALGORITHM = "VECTOR_INDEX_ALGORITHM"; + public static final byte[] VECTOR_INDEX_ALGORITHM_BYTES = Bytes.toBytes(VECTOR_INDEX_ALGORITHM); + + public static final String VECTOR_DISTANCE_METRIC = "VECTOR_DISTANCE_METRIC"; + public static final byte[] VECTOR_DISTANCE_METRIC_BYTES = Bytes.toBytes(VECTOR_DISTANCE_METRIC); + + public static final String VECTOR_DIMENSION = "VECTOR_DIMENSION"; + public static final byte[] VECTOR_DIMENSION_BYTES = Bytes.toBytes(VECTOR_DIMENSION); + + public static final String VECTOR_IVF_LISTS = "VECTOR_IVF_LISTS"; + public static final byte[] VECTOR_IVF_LISTS_BYTES = Bytes.toBytes(VECTOR_IVF_LISTS); + + public static final String VECTOR_IVF_SAMPLE_SIZE = "VECTOR_IVF_SAMPLE_SIZE"; + public static final byte[] VECTOR_IVF_SAMPLE_SIZE_BYTES = Bytes.toBytes(VECTOR_IVF_SAMPLE_SIZE); + + public static final String VECTOR_CENTROID_GENERATION = "VECTOR_CENTROID_GENERATION"; + public static final byte[] VECTOR_CENTROID_GENERATION_BYTES = + Bytes.toBytes(VECTOR_CENTROID_GENERATION); + public static final String SYSTEM_CHILD_LINK_TABLE = "CHILD_LINK"; public static final String SYSTEM_CHILD_LINK_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_CHILD_LINK_TABLE); @@ -510,6 +529,22 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final String TRACKER_STATUS_COMPLETE = "c"; public static final int IDX_CDC_TRACKER_TTL = 10 * 24 * 60 * 60; + public static final String SYSTEM_VECTOR_CENTROID_TABLE = "VECTOR_CENTROID"; + public static final String SYSTEM_VECTOR_CENTROID_NAME = + SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_VECTOR_CENTROID_TABLE); + public static final byte[] SYSTEM_VECTOR_CENTROID_NAME_BYTES = + Bytes.toBytes(SYSTEM_VECTOR_CENTROID_NAME); + public static final TableName SYSTEM_VECTOR_CENTROID_HBASE_TABLE_NAME = + TableName.valueOf(SYSTEM_VECTOR_CENTROID_NAME); + public static final String INDEX_NAME = "INDEX_NAME"; + public static final byte[] INDEX_NAME_BYTES = Bytes.toBytes(INDEX_NAME); + public static final String CENTROID_ID = "CENTROID_ID"; + public static final byte[] CENTROID_ID_BYTES = Bytes.toBytes(CENTROID_ID); + public static final String CENTROID_VECTOR = "CENTROID_VECTOR"; + public static final byte[] CENTROID_VECTOR_BYTES = Bytes.toBytes(CENTROID_VECTOR); + public static final String GENERATION_ID = "GENERATION_ID"; + public static final byte[] GENERATION_ID_BYTES = Bytes.toBytes(GENERATION_ID); + public static final String QUERY_ID = "QUERY_ID"; public static final String USER = "USER"; public static final String CLIENT_IP = "CLIENT_IP"; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index f14ae7aa71..bd24c7b392 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -4139,6 +4139,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices return setSystemDDLProperties(QueryConstants.CREATE_IDX_CDC_TRACKER_METADATA); } + protected String getVectorCentroidDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_VECTOR_CENTROID_METADATA); + } + private String setSystemDDLProperties(String ddl) { return String.format(ddl, props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, @@ -4476,6 +4480,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices metaConnection.createStatement().executeUpdate(getIdxCdcTrackerDDL()); } catch (TableAlreadyExistsException ignore) { } + try { + metaConnection.createStatement().executeUpdate(getVectorCentroidDDL()); + } catch (TableAlreadyExistsException ignore) { + } try { // check if we have old PHOENIX_INDEX_TOOL tables // move data to the new tables under System, or simply create the new tables @@ -4803,22 +4811,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices } if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0) { metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 9, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 15, PhoenixDatabaseMetaData.PHYSICAL_TABLE_NAME + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 8, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 14, PhoenixDatabaseMetaData.SCHEMA_VERSION + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 7, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 13, PhoenixDatabaseMetaData.EXTERNAL_SCHEMA_ID + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 6, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 12, PhoenixDatabaseMetaData.STREAMING_TOPIC_NAME + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 5, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 11, PhoenixDatabaseMetaData.INDEX_WHERE + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 4, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 10, PhoenixDatabaseMetaData.CDC_INCLUDE_TABLE + " " + PVarchar.INSTANCE.getSqlTypeName()); /** @@ -4826,16 +4834,35 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices * PHOENIX_TTL Column. See PHOENIX-7023 */ metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 3, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 9, PhoenixDatabaseMetaData.TTL + " " + PVarchar.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 2, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 8, PhoenixDatabaseMetaData.ROW_KEY_MATCHER + " " + PVarbinary.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 1, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 7, PhoenixDatabaseMetaData.IS_STRICT_TTL + " " + PBoolean.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0, PhoenixDatabaseMetaData.INDEX_CONSISTENCY + " CHAR(1)"); + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 6, + PhoenixDatabaseMetaData.INDEX_CONSISTENCY + " CHAR(1)"); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 5, + PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM + " " + PVarchar.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 4, + PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC + " " + PVarchar.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 3, + PhoenixDatabaseMetaData.VECTOR_DIMENSION + " " + PInteger.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 2, + PhoenixDatabaseMetaData.VECTOR_IVF_LISTS + " " + PInteger.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0 - 1, + PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE + " " + PInteger.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MIN_SYSTEM_TABLE_TIMESTAMP_5_4_0, + PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION + " " + PLong.INSTANCE.getSqlTypeName()); // move TTL values stored in descriptor to SYSCAT TTL column. moveTTLFromHBaseLevelTTLToPhoenixLevelTTL(metaConnection); @@ -5063,6 +5090,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices metaConnection = upgradeSystemCDCStreamStatus(metaConnection); metaConnection = upgradeSystemCDCStream(metaConnection); metaConnection = upgradeSystemIdxCdcTracker(metaConnection); + metaConnection = upgradeSystemVectorCentroid(metaConnection); // As this is where the most time will be spent during an upgrade, // especially when there are large number of views. @@ -5394,6 +5422,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices return metaConnection; } + private PhoenixConnection upgradeSystemVectorCentroid(PhoenixConnection metaConnection) + throws SQLException { + try { + metaConnection.createStatement().executeUpdate(getVectorCentroidDDL()); + } catch (TableAlreadyExistsException ignored) { + } + return metaConnection; + } + // Special method for adding the column qualifier column for 4.10. private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index abbc08ac31..c8a737c978 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -223,6 +223,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices return setSystemDDLProperties(QueryConstants.CREATE_IDX_CDC_TRACKER_METADATA); } + protected String getVectorCentroidDDL() { + return setSystemDDLProperties(QueryConstants.CREATE_VECTOR_CENTROID_METADATA); + } + private String setSystemDDLProperties(String ddl) { return String.format(ddl, props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, @@ -504,6 +508,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices metaConnection.createStatement().executeUpdate(getIdxCdcTrackerDDL()); } catch (TableAlreadyExistsException ignore) { } + try { + metaConnection.createStatement().executeUpdate(getVectorCentroidDDL()); + } catch (TableAlreadyExistsException ignore) { + } } catch (SQLException e) { sqlE = e; } finally { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java index aa73c83314..c5f5423412 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -26,6 +26,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BIND_PARAMETERS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.BUFFER_LENGTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CDC_INCLUDE_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_VECTOR; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHANGE_DETECTION_ENABLED; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME; @@ -50,6 +52,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXCEPTION_TRACE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXPLAIN_PLAN; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.EXTERNAL_SCHEMA_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GENERATION_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GLOBAL_SCAN_DETAILS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; @@ -60,6 +63,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_CONSISTENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_WHERE; @@ -143,6 +147,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_TRANSFORM_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; @@ -174,6 +179,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DIMENSION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE; @@ -419,7 +430,10 @@ public interface QueryConstants { + " BOOLEAN, \n" + SCHEMA_VERSION + " VARCHAR, \n" + EXTERNAL_SCHEMA_ID + " VARCHAR, \n" + STREAMING_TOPIC_NAME + " VARCHAR, \n" + INDEX_WHERE + " VARCHAR, \n" + CDC_INCLUDE_TABLE + " VARCHAR, \n" + TTL + " VARCHAR, \n" + ROW_KEY_MATCHER + " VARBINARY_ENCODED, \n" - + IS_STRICT_TTL + " BOOLEAN, \n" + INDEX_CONSISTENCY + " CHAR(1), \n" + + + IS_STRICT_TTL + " BOOLEAN, \n" + INDEX_CONSISTENCY + " CHAR(1), \n" + VECTOR_INDEX_ALGORITHM + + " VARCHAR, \n" + VECTOR_DISTANCE_METRIC + " VARCHAR, \n" + VECTOR_DIMENSION + " INTEGER, \n" + + VECTOR_IVF_LISTS + " INTEGER, \n" + VECTOR_IVF_SAMPLE_SIZE + " INTEGER, \n" + + VECTOR_CENTROID_GENERATION + " BIGINT, \n" + // Column metadata (will be null for table row) DATA_TYPE + " INTEGER," + COLUMN_SIZE + " INTEGER," + DECIMAL_DIGITS + " INTEGER," + NULLABLE + " INTEGER," + ORDINAL_POSITION + " INTEGER," + SORT_ORDER + " INTEGER," + ARRAY_SIZE @@ -611,4 +625,15 @@ public interface QueryConstants { + Boolean.FALSE + ",\n" + UPDATE_CACHE_FREQUENCY + "=" + "7200000" + ",\n" + ColumnFamilyDescriptorBuilder.TTL + "=" + IDX_CDC_TRACKER_TTL + ",\n" + "\"phoenix.max.lookback.age.seconds\"=0"; + + String CREATE_VECTOR_CENTROID_METADATA = + "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_VECTOR_CENTROID_TABLE + "\"(\n" + + // PK columns + INDEX_NAME + " VARCHAR NOT NULL," + GENERATION_ID + " BIGINT NOT NULL," + CENTROID_ID + + " INTEGER NOT NULL,\n" + + // Non-PK columns + CENTROID_VECTOR + " VARBINARY,\n" + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + + INDEX_NAME + "," + GENERATION_ID + "," + CENTROID_ID + "))\n" + HConstants.VERSIONS + + "=%s,\n" + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS + "=%s,\n" + TRANSACTIONAL + "=" + + Boolean.FALSE; } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java index b41da68782..8a9a3adda0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -480,4 +480,34 @@ public class DelegateTable implements PTable { public Map<String, String> getDefaultPropertyValues() { return delegate.getDefaultPropertyValues(); } + + @Override + public String getVectorIndexAlgorithm() { + return delegate.getVectorIndexAlgorithm(); + } + + @Override + public String getVectorDistanceMetric() { + return delegate.getVectorDistanceMetric(); + } + + @Override + public Integer getVectorDimension() { + return delegate.getVectorDimension(); + } + + @Override + public Integer getVectorIvfLists() { + return delegate.getVectorIvfLists(); + } + + @Override + public Integer getVectorIvfSampleSize() { + return delegate.getVectorIvfSampleSize(); + } + + @Override + public Long getVectorCentroidGeneration() { + return delegate.getVectorCentroidGeneration(); + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java index effd7773ec..dc12bc1569 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java @@ -108,7 +108,8 @@ public interface PTable extends PMetaDataEntity { public enum IndexType { GLOBAL((byte) 1), // Covered Global LOCAL((byte) 2), // Covered Local - UNCOVERED_GLOBAL((byte) 3); // Uncovered Global + UNCOVERED_GLOBAL((byte) 3), // Uncovered Global + VECTOR_GLOBAL((byte) 4); private final byte[] byteValue; private final byte serializedValue; @@ -1062,6 +1063,32 @@ public interface PTable extends PMetaDataEntity { */ byte[] getRowKeyMatcher(); + /** Returns the vector index algorithm (for example IVF), or null if not a vector index. */ + String getVectorIndexAlgorithm(); + + /** Returns the distance metric of the vector index, or null if not a vector index. */ + String getVectorDistanceMetric(); + + /** Returns the dimension of the indexed vectors, or null if not a vector index. */ + Integer getVectorDimension(); + + /** Returns the number of IVF lists (partitions), or null if not a vector index. */ + Integer getVectorIvfLists(); + + /** Returns the number of sample vectors for IVF training, or null if not a vector index. */ + Integer getVectorIvfSampleSize(); + + /** + * Returns the ID of the active centroid generation. Returns null if this is not a vector index, + * or if the index does not have an active generation yet. + */ + Long getVectorCentroidGeneration(); + + /** Returns true if this table is a vector index. */ + default boolean isVectorIndex() { + return getIndexType() == IndexType.VECTOR_GLOBAL; + } + /** * Class to help track encoded column qualifier counters per column family. */ diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java index c1c1f13c7d..45667de0ea 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -224,6 +224,12 @@ public class PTableImpl implements PTable { private Set<ColumnReference> indexWhereColumns; private Map<PTableKey, Long> ancestorLastDDLTimestampMap; private Set<CDCChangeScope> cdcIncludeScopes; + private final String vectorIndexAlgorithm; + private final String vectorDistanceMetric; + private final Integer vectorDimension; + private final Integer vectorIvfLists; + private final Integer vectorIvfSampleSize; + private final Long vectorCentroidGeneration; public static class Builder { private PTableKey key; @@ -294,6 +300,12 @@ public class PTableImpl implements PTable { private Map<PTableKey, Long> ancestorLastDDLTimestampMap = new HashMap<>(); private TTLExpression ttl = TTL_EXPRESSION_NOT_DEFINED; private byte[] rowKeyMatcher; + private String vectorIndexAlgorithm; + private String vectorDistanceMetric; + private Integer vectorDimension; + private Integer vectorIvfLists; + private Integer vectorIvfSampleSize; + private Long vectorCentroidGeneration; // Used to denote which properties a view has explicitly modified private BitSet viewModifiedPropSet = new BitSet(3); @@ -745,6 +757,36 @@ public class PTableImpl implements PTable { return this; } + public Builder setVectorIndexAlgorithm(String vectorIndexAlgorithm) { + this.vectorIndexAlgorithm = vectorIndexAlgorithm; + return this; + } + + public Builder setVectorDistanceMetric(String vectorDistanceMetric) { + this.vectorDistanceMetric = vectorDistanceMetric; + return this; + } + + public Builder setVectorDimension(Integer vectorDimension) { + this.vectorDimension = vectorDimension; + return this; + } + + public Builder setVectorIvfLists(Integer vectorIvfLists) { + this.vectorIvfLists = vectorIvfLists; + return this; + } + + public Builder setVectorIvfSampleSize(Integer vectorIvfSampleSize) { + this.vectorIvfSampleSize = vectorIvfSampleSize; + return this; + } + + public Builder setVectorCentroidGeneration(Long vectorCentroidGeneration) { + this.vectorCentroidGeneration = vectorCentroidGeneration; + return this; + } + /** * Populate derivable attributes of the PTable * @return PTableImpl.Builder object @@ -1028,6 +1070,12 @@ public class PTableImpl implements PTable { this.indexWhere = builder.indexWhere; this.ancestorLastDDLTimestampMap = builder.ancestorLastDDLTimestampMap; this.rowKeyMatcher = builder.rowKeyMatcher; + this.vectorIndexAlgorithm = builder.vectorIndexAlgorithm; + this.vectorDistanceMetric = builder.vectorDistanceMetric; + this.vectorDimension = builder.vectorDimension; + this.vectorIvfLists = builder.vectorIvfLists; + this.vectorIvfSampleSize = builder.vectorIvfSampleSize; + this.vectorCentroidGeneration = builder.vectorCentroidGeneration; } // When cloning table, ignore the salt column as it will be added back in the constructor @@ -1100,7 +1148,12 @@ public class PTableImpl implements PTable { .setStreamingTopicName(table.getStreamingTopicName()).setIndexWhere(table.getIndexWhere()) .setCDCIncludeScopes(table.getCDCIncludeScopes()) .setAncestorLastDDLTimestampMap(table.getAncestorLastDDLTimestampMap()) - .setTTL(table.getTTLExpression()).setRowKeyMatcher(table.getRowKeyMatcher()); + .setTTL(table.getTTLExpression()).setRowKeyMatcher(table.getRowKeyMatcher()) + .setVectorIndexAlgorithm(table.getVectorIndexAlgorithm()) + .setVectorDistanceMetric(table.getVectorDistanceMetric()) + .setVectorDimension(table.getVectorDimension()).setVectorIvfLists(table.getVectorIvfLists()) + .setVectorIvfSampleSize(table.getVectorIvfSampleSize()) + .setVectorCentroidGeneration(table.getVectorCentroidGeneration()); } @Override @@ -2111,6 +2164,30 @@ public class PTableImpl implements PTable { if (table.hasRowKeyMatcher()) { rowKeyMatcher = table.getRowKeyMatcher().toByteArray(); } + String vectorIndexAlgorithm = null; + if (table.hasVectorIndexAlgorithm()) { + vectorIndexAlgorithm = table.getVectorIndexAlgorithm(); + } + String vectorDistanceMetric = null; + if (table.hasVectorDistanceMetric()) { + vectorDistanceMetric = table.getVectorDistanceMetric(); + } + Integer vectorDimension = null; + if (table.hasVectorDimension()) { + vectorDimension = table.getVectorDimension(); + } + Integer vectorIvfLists = null; + if (table.hasVectorIvfLists()) { + vectorIvfLists = table.getVectorIvfLists(); + } + Integer vectorIvfSampleSize = null; + if (table.hasVectorIvfSampleSize()) { + vectorIvfSampleSize = table.getVectorIvfSampleSize(); + } + Long vectorCentroidGeneration = null; + if (table.hasVectorCentroidGeneration()) { + vectorCentroidGeneration = table.getVectorCentroidGeneration(); + } IndexConsistency indexConsistency = null; if (tableType == PTableType.INDEX) { if (table.hasIndexConsistency()) { @@ -2154,7 +2231,11 @@ public class PTableImpl implements PTable { .setIsStrictTTL(isStrictTTL).setSchemaVersion(schemaVersion) .setExternalSchemaId(externalSchemaId).setStreamingTopicName(streamingTopicName) .setCDCIncludeScopes(CDCUtil.makeChangeScopeEnumsFromString(cdcIncludeScopesStr)) - .setIndexWhere(indexWhere).setTTL(ttl).setRowKeyMatcher(rowKeyMatcher).build(); + .setIndexWhere(indexWhere).setTTL(ttl).setRowKeyMatcher(rowKeyMatcher) + .setVectorIndexAlgorithm(vectorIndexAlgorithm).setVectorDistanceMetric(vectorDistanceMetric) + .setVectorDimension(vectorDimension).setVectorIvfLists(vectorIvfLists) + .setVectorIvfSampleSize(vectorIvfSampleSize) + .setVectorCentroidGeneration(vectorCentroidGeneration).build(); } catch (SQLException e) { throw new RuntimeException(e); // Impossible } @@ -2327,6 +2408,24 @@ public class PTableImpl implements PTable { if (table.getRowKeyMatcher() != null) { builder.setRowKeyMatcher(ByteStringer.wrap(table.getRowKeyMatcher())); } + if (table.getVectorIndexAlgorithm() != null) { + builder.setVectorIndexAlgorithm(table.getVectorIndexAlgorithm()); + } + if (table.getVectorDistanceMetric() != null) { + builder.setVectorDistanceMetric(table.getVectorDistanceMetric()); + } + if (table.getVectorDimension() != null) { + builder.setVectorDimension(table.getVectorDimension()); + } + if (table.getVectorIvfLists() != null) { + builder.setVectorIvfLists(table.getVectorIvfLists()); + } + if (table.getVectorIvfSampleSize() != null) { + builder.setVectorIvfSampleSize(table.getVectorIvfSampleSize()); + } + if (table.getVectorCentroidGeneration() != null) { + builder.setVectorCentroidGeneration(table.getVectorCentroidGeneration()); + } return builder.build(); } @@ -2570,6 +2669,36 @@ public class PTableImpl implements PTable { return rowKeyMatcher; } + @Override + public String getVectorIndexAlgorithm() { + return vectorIndexAlgorithm; + } + + @Override + public String getVectorDistanceMetric() { + return vectorDistanceMetric; + } + + @Override + public Integer getVectorDimension() { + return vectorDimension; + } + + @Override + public Integer getVectorIvfLists() { + return vectorIvfLists; + } + + @Override + public Integer getVectorIvfSampleSize() { + return vectorIvfSampleSize; + } + + @Override + public Long getVectorCentroidGeneration() { + return vectorCentroidGeneration; + } + private static final class KVColumnFamilyQualifier { @Nonnull private final String colFamilyName; diff --git a/phoenix-core-client/src/main/protobuf/PTable.proto b/phoenix-core-client/src/main/protobuf/PTable.proto index a98239f9ed..d2c4496d75 100644 --- a/phoenix-core-client/src/main/protobuf/PTable.proto +++ b/phoenix-core-client/src/main/protobuf/PTable.proto @@ -125,6 +125,12 @@ message PTable { optional bytes rowKeyMatcher = 56; optional bool isStrictTTL = 57; optional string indexConsistency = 58; + optional string vectorIndexAlgorithm = 59; + optional string vectorDistanceMetric = 60; + optional int32 vectorDimension = 61; + optional int32 vectorIvfLists = 62; + optional int32 vectorIvfSampleSize = 63; + optional int64 vectorCentroidGeneration = 64; } message EncodedCQCounter { diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 053976b586..5b42865d20 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -81,6 +81,12 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DIMENSION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES; @@ -219,6 +225,7 @@ import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.mapreduce.util.ConnectionUtil; import org.apache.phoenix.metrics.Metrics; +import org.apache.phoenix.optimize.DistanceMetric; import org.apache.phoenix.parse.LiteralParseNode; import org.apache.phoenix.parse.PFunction; import org.apache.phoenix.parse.PFunction.FunctionArgument; @@ -423,6 +430,18 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, INDEX_WHERE_BYTES); private static final Cell INDEX_CONSISTENCY_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, INDEX_CONSISTENCY_BYTES); + private static final Cell VECTOR_INDEX_ALGORITHM_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_INDEX_ALGORITHM_BYTES); + private static final Cell VECTOR_DISTANCE_METRIC_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_DISTANCE_METRIC_BYTES); + private static final Cell VECTOR_DIMENSION_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_DIMENSION_BYTES); + private static final Cell VECTOR_IVF_LISTS_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_IVF_LISTS_BYTES); + private static final Cell VECTOR_IVF_SAMPLE_SIZE_KV = + createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_IVF_SAMPLE_SIZE_BYTES); + private static final Cell VECTOR_CENTROID_GENERATION_KV = createFirstOnRow( + ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, VECTOR_CENTROID_GENERATION_BYTES); private static final Cell TTL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TTL_BYTES); @@ -441,7 +460,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr AUTO_PARTITION_SEQ_KV, APPEND_ONLY_SCHEMA_KV, STORAGE_SCHEME_KV, ENCODING_SCHEME_KV, USE_STATS_FOR_PARALLELIZATION_KV, LAST_DDL_TIMESTAMP_KV, CHANGE_DETECTION_ENABLED_KV, SCHEMA_VERSION_KV, EXTERNAL_SCHEMA_ID_KV, STREAMING_TOPIC_NAME_KV, INDEX_WHERE_KV, - CDC_INCLUDE_KV, TTL_KV, ROW_KEY_MATCHER_KV, IS_STRICT_TTL_KV, INDEX_CONSISTENCY_KV); + CDC_INCLUDE_KV, TTL_KV, ROW_KEY_MATCHER_KV, IS_STRICT_TTL_KV, INDEX_CONSISTENCY_KV, + VECTOR_INDEX_ALGORITHM_KV, VECTOR_DISTANCE_METRIC_KV, VECTOR_DIMENSION_KV, VECTOR_IVF_LISTS_KV, + VECTOR_IVF_SAMPLE_SIZE_KV, VECTOR_CENTROID_GENERATION_KV); static { Collections.sort(TABLE_KV_COLUMNS, CellComparatorImpl.COMPARATOR); @@ -504,6 +525,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr private static final int ROW_KEY_MATCHER_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_MATCHER_KV); private static final int IS_STRICT_TTL_INDEX = TABLE_KV_COLUMNS.indexOf(IS_STRICT_TTL_KV); private static final int INDEX_CONSISTENCY_INDEX = TABLE_KV_COLUMNS.indexOf(INDEX_CONSISTENCY_KV); + private static final int VECTOR_INDEX_ALGORITHM_INDEX = + TABLE_KV_COLUMNS.indexOf(VECTOR_INDEX_ALGORITHM_KV); + private static final int VECTOR_DISTANCE_METRIC_INDEX = + TABLE_KV_COLUMNS.indexOf(VECTOR_DISTANCE_METRIC_KV); + private static final int VECTOR_DIMENSION_INDEX = TABLE_KV_COLUMNS.indexOf(VECTOR_DIMENSION_KV); + private static final int VECTOR_IVF_LISTS_INDEX = TABLE_KV_COLUMNS.indexOf(VECTOR_IVF_LISTS_KV); + private static final int VECTOR_IVF_SAMPLE_SIZE_INDEX = + TABLE_KV_COLUMNS.indexOf(VECTOR_IVF_SAMPLE_SIZE_KV); + private static final int VECTOR_CENTROID_GENERATION_INDEX = + TABLE_KV_COLUMNS.indexOf(VECTOR_CENTROID_GENERATION_KV); // KeyValues for Column private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES); @@ -1603,6 +1634,70 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr : oldTable != null ? oldTable.getIndexConsistency() : null); + Cell vectorIndexAlgorithmKv = tableKeyValues[VECTOR_INDEX_ALGORITHM_INDEX]; + String vectorIndexAlgorithm = null; + if (vectorIndexAlgorithmKv != null) { + vectorIndexAlgorithm = + (String) PVarchar.INSTANCE.toObject(vectorIndexAlgorithmKv.getValueArray(), + vectorIndexAlgorithmKv.getValueOffset(), vectorIndexAlgorithmKv.getValueLength()); + } + builder.setVectorIndexAlgorithm(vectorIndexAlgorithm != null ? vectorIndexAlgorithm + : oldTable != null ? oldTable.getVectorIndexAlgorithm() + : null); + + Cell vectorDistanceMetricKv = tableKeyValues[VECTOR_DISTANCE_METRIC_INDEX]; + String vectorDistanceMetric = null; + if (vectorDistanceMetricKv != null) { + vectorDistanceMetric = + (String) PVarchar.INSTANCE.toObject(vectorDistanceMetricKv.getValueArray(), + vectorDistanceMetricKv.getValueOffset(), vectorDistanceMetricKv.getValueLength()); + } + builder.setVectorDistanceMetric(vectorDistanceMetric != null ? vectorDistanceMetric + : oldTable != null ? oldTable.getVectorDistanceMetric() + : null); + + Cell vectorDimensionKv = tableKeyValues[VECTOR_DIMENSION_INDEX]; + Integer vectorDimension = null; + if (vectorDimensionKv != null) { + vectorDimension = (Integer) PInteger.INSTANCE.toObject(vectorDimensionKv.getValueArray(), + vectorDimensionKv.getValueOffset(), vectorDimensionKv.getValueLength()); + } + builder.setVectorDimension(vectorDimension != null ? vectorDimension + : oldTable != null ? oldTable.getVectorDimension() + : null); + + Cell vectorIvfListsKv = tableKeyValues[VECTOR_IVF_LISTS_INDEX]; + Integer vectorIvfLists = null; + if (vectorIvfListsKv != null) { + vectorIvfLists = (Integer) PInteger.INSTANCE.toObject(vectorIvfListsKv.getValueArray(), + vectorIvfListsKv.getValueOffset(), vectorIvfListsKv.getValueLength()); + } + builder.setVectorIvfLists(vectorIvfLists != null ? vectorIvfLists + : oldTable != null ? oldTable.getVectorIvfLists() + : null); + + Cell vectorIvfSampleSizeKv = tableKeyValues[VECTOR_IVF_SAMPLE_SIZE_INDEX]; + Integer vectorIvfSampleSize = null; + if (vectorIvfSampleSizeKv != null) { + vectorIvfSampleSize = + (Integer) PInteger.INSTANCE.toObject(vectorIvfSampleSizeKv.getValueArray(), + vectorIvfSampleSizeKv.getValueOffset(), vectorIvfSampleSizeKv.getValueLength()); + } + builder.setVectorIvfSampleSize(vectorIvfSampleSize != null ? vectorIvfSampleSize + : oldTable != null ? oldTable.getVectorIvfSampleSize() + : null); + + Cell vectorCentroidGenerationKv = tableKeyValues[VECTOR_CENTROID_GENERATION_INDEX]; + Long vectorCentroidGeneration = null; + if (vectorCentroidGenerationKv != null) { + vectorCentroidGeneration = + (Long) PLong.INSTANCE.toObject(vectorCentroidGenerationKv.getValueArray(), + vectorCentroidGenerationKv.getValueOffset(), vectorCentroidGenerationKv.getValueLength()); + } + builder.setVectorCentroidGeneration(vectorCentroidGeneration != null ? vectorCentroidGeneration + : oldTable != null ? oldTable.getVectorCentroidGeneration() + : null); + // Check the cell tag to see whether the view has modified this property final byte[] tagUseStatsForParallelization = (useStatsForParallelizationKv == null) ? HConstants.EMPTY_BYTE_ARRAY @@ -2459,6 +2554,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr request.hasParentTable() ? PTableImpl.createFromProto(request.getParentTable()) : null; PTableType tableType = MetaDataUtil.getTableType(tableMetadata, GenericKeyValueBuilder.INSTANCE, new ImmutableBytesWritable()); + validateVectorIndexMetadata(tableMetadata, tableType, indexType); // Load table to see if it already exists byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaName, tableName); @@ -2962,6 +3058,57 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements RegionCopr } } + /** + * Checks the vector index metadata before the server writes it to SYSTEM.CATALOG. + * <p> + * If the table is not a VECTOR_GLOBAL index and has no vector algorithm, the check does nothing. + * Only a VECTOR_GLOBAL index can have vector metadata. The algorithm must be IVF, and the + * distance metric must be a known {@link DistanceMetric}. The IVF list count must be more than + * zero, and the sample size must be equal to or more than the list count. A failed check throws a + * SQLException with the applicable vector error code. + */ + private static void validateVectorIndexMetadata(List<Mutation> tableMetadata, + PTableType tableType, IndexType indexType) throws SQLException { + Put header = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetadata); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + KeyValueBuilder kvBuilder = GenericKeyValueBuilder.INSTANCE; + String algorithm = header != null + && MetaDataUtil.getMutationValue(header, VECTOR_INDEX_ALGORITHM_BYTES, kvBuilder, ptr) + ? (String) PVarchar.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()) + : null; + if (indexType != IndexType.VECTOR_GLOBAL && algorithm == null) { + return; + } + if (tableType != PTableType.INDEX || indexType != IndexType.VECTOR_GLOBAL) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("Vector index metadata is only valid on a VECTOR_GLOBAL index").build() + .buildException(); + } + if (!"IVF".equalsIgnoreCase(algorithm)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_VECTOR_INDEX_ALGORITHM) + .setMessage(String.valueOf(algorithm)).build().buildException(); + } + String metric = + MetaDataUtil.getMutationValue(header, VECTOR_DISTANCE_METRIC_BYTES, kvBuilder, ptr) + ? (String) PVarchar.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()) + : null; + if (metric == null || DistanceMetric.fromString(metric) == null) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC) + .setMessage(String.valueOf(metric)).build().buildException(); + } + Integer lists = MetaDataUtil.getMutationValue(header, VECTOR_IVF_LISTS_BYTES, kvBuilder, ptr) + ? (Integer) PInteger.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()) + : null; + Integer sampleSize = + MetaDataUtil.getMutationValue(header, VECTOR_IVF_SAMPLE_SIZE_BYTES, kvBuilder, ptr) + ? (Integer) PInteger.INSTANCE.toObject(ptr.get(), ptr.getOffset(), ptr.getLength()) + : null; + if (lists == null || lists <= 0 || sampleSize == null || sampleSize < lists) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS) + .setMessage("lists=" + lists + ", sample_size=" + sampleSize).build().buildException(); + } + } + private void exportSchema(List<Mutation> tableMetadata, byte[] tableKey, long clientTimestamp, int clientVersion, PTable oldTable) throws SQLException, IOException { List<Cell> tableCellList = MetaDataUtil.getTableCellsFromMutations(tableMetadata); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java index ac0002a479..324e1b13b3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java @@ -68,14 +68,16 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest { private static final Set<String> HBASE_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(Arrays.asList("SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL")); - private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>( - Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", - "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", - "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER")); - private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>( - Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", - "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM", - "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER")); + private static final Set<String> PHOENIX_SYSTEM_TABLES = + new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", + "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", + "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER", + "SYSTEM.VECTOR_CENTROID")); + private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = + new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", + "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", + "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER", + "SYSTEM:VECTOR_CENTROID")); private static final String SCHEMA_NAME = "MIGRATETEST"; private static final String TABLE_NAME = SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase(); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java index 3568628ec7..089226e56b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipSystemTablesExistenceCheckIT.java @@ -68,7 +68,7 @@ public class SkipSystemTablesExistenceCheckIT { new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER", - "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL")); + "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL", "SYSTEM.VECTOR_CENTROID")); private static class PhoenixSystemTablesCreationTestDriver extends PhoenixTestDriver { private static ConnectionQueryServices cqs; diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java index 5d1cb86a2b..b7c5e08fda 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java @@ -108,13 +108,13 @@ public class SystemTablesCreationOnConnectionIT { new HashSet<>(Arrays.asList("SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION", "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK", "SYSTEM.TRANSFORM", "SYSTEM.CDC_STREAM_STATUS", "SYSTEM.CDC_STREAM", "SYSTEM.IDX_CDC_TRACKER", - "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL")); + "SYSTEM.VECTOR_CENTROID", "SYSTEM.PHOENIX_INDEX_TOOL_RESULT", "SYSTEM.PHOENIX_INDEX_TOOL")); private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION", "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK", "SYSTEM:TRANSFORM", "SYSTEM:CDC_STREAM_STATUS", "SYSTEM:CDC_STREAM", "SYSTEM:IDX_CDC_TRACKER", - "SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL")); + "SYSTEM:VECTOR_CENTROID", "SYSTEM:PHOENIX_INDEX_TOOL_RESULT", "SYSTEM:PHOENIX_INDEX_TOOL")); private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorCentroidTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorCentroidTableIT.java new file mode 100644 index 0000000000..1192fe3fca --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorCentroidTableIT.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CENTROID_VECTOR; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GENERATION_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_VECTOR_CENTROID_TABLE; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.sql.Types; +import java.util.List; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVarbinary; +import org.apache.phoenix.schema.types.PVarchar; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the SYSTEM.VECTOR_CENTROID schema and primary key, and the lifecycle of centroid rows over + * more than one generation: persist, load, delete, generation summary, scorecard, and rebuild + * claim. + */ +@Category(ParallelStatsDisabledTest.class) +public class VectorCentroidTableIT extends ParallelStatsDisabledIT { + + /** + * Tests that the table exists and has 12 columns. Tests the order, SQL type, and nullability of + * the key and vector columns in the result set metadata. + */ + @Test + public void testTableExistenceAndSchema() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement(); ResultSet rs = + stmt.executeQuery("SELECT * FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE 1=0")) { + ResultSetMetaData rsmd = rs.getMetaData(); + assertEquals(4, rsmd.getColumnCount()); + + assertEquals(INDEX_NAME, rsmd.getColumnName(1)); + assertEquals(Types.VARCHAR, rsmd.getColumnType(1)); + assertEquals(ResultSetMetaData.columnNoNulls, rsmd.isNullable(1)); + + assertEquals(GENERATION_ID, rsmd.getColumnName(2)); + assertEquals(Types.BIGINT, rsmd.getColumnType(2)); + assertEquals(ResultSetMetaData.columnNoNulls, rsmd.isNullable(2)); + + assertEquals(CENTROID_ID, rsmd.getColumnName(3)); + assertEquals(Types.INTEGER, rsmd.getColumnType(3)); + assertEquals(ResultSetMetaData.columnNoNulls, rsmd.isNullable(3)); + + assertEquals(CENTROID_VECTOR, rsmd.getColumnName(4)); + assertEquals(Types.VARBINARY, rsmd.getColumnType(4)); + assertEquals(ResultSetMetaData.columnNullable, rsmd.isNullable(4)); + } + } + } + + /** Tests that an upsert replaces the row with the same key and keeps generations separate. */ + @Test + public void testPrimaryKeyConstraintAndUpsert() throws Exception { + String indexName = "TEST_VECTOR_IDX_" + generateUniqueName(); + byte[] vector1 = new byte[] { 1, 2, 3, 4 }; + byte[] vector2 = new byte[] { 5, 6, 7, 8 }; + byte[] vector3 = new byte[] { 9, 10, 11, 12 }; + + try (Connection conn = DriverManager.getConnection(getUrl())) { + String upsertSql = "UPSERT INTO " + SYSTEM_VECTOR_CENTROID_NAME + " (" + INDEX_NAME + ", " + + GENERATION_ID + ", " + CENTROID_ID + ", " + CENTROID_VECTOR + ") VALUES (?, ?, ?, ?)"; + + try (PreparedStatement ps = conn.prepareStatement(upsertSql)) { + ps.setString(1, indexName); + ps.setLong(2, 1L); + ps.setInt(3, 0); + ps.setBytes(4, vector1); + ps.executeUpdate(); + } + conn.commit(); + + // An upsert with an existing (INDEX_NAME, GENERATION_ID, CENTROID_ID) key replaces the + // centroid vector + try (PreparedStatement ps = conn.prepareStatement(upsertSql)) { + ps.setString(1, indexName); + ps.setLong(2, 1L); + ps.setInt(3, 0); + ps.setBytes(4, vector2); + ps.executeUpdate(); + } + conn.commit(); + + String countSql = + "SELECT COUNT(*) FROM " + SYSTEM_VECTOR_CENTROID_NAME + " WHERE " + INDEX_NAME + " = ?"; + try (PreparedStatement ps = conn.prepareStatement(countSql)) { + ps.setString(1, indexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals("Expected single row after overwriting PK", 1, rs.getInt(1)); + } + } + + String selectSql = "SELECT " + CENTROID_VECTOR + " FROM " + SYSTEM_VECTOR_CENTROID_NAME + + " WHERE " + INDEX_NAME + " = ? AND " + GENERATION_ID + " = ? AND " + CENTROID_ID + " = ?"; + try (PreparedStatement ps = conn.prepareStatement(selectSql)) { + ps.setString(1, indexName); + ps.setLong(2, 1L); + ps.setInt(3, 0); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertArrayEquals(vector2, rs.getBytes(1)); + } + } + + // Centroids of different generations of the same index are separate rows + try (PreparedStatement ps = conn.prepareStatement(upsertSql)) { + ps.setString(1, indexName); + ps.setLong(2, 2L); + ps.setInt(3, 0); + ps.setBytes(4, vector3); + ps.executeUpdate(); + } + conn.commit(); + + try (PreparedStatement ps = conn.prepareStatement(countSql)) { + ps.setString(1, indexName); + try (ResultSet rs = ps.executeQuery()) { + assertTrue(rs.next()); + assertEquals("Expected two rows for distinct generations", 2, rs.getInt(1)); + } + } + } + } + + /** Tests that the PTable shows the correct primary key columns and centroid vector column. */ + @Test + public void testPTableSchema() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + PTable table = pconn.getTableNoCache(SYSTEM_VECTOR_CENTROID_NAME); + assertNotNull("SYSTEM.VECTOR_CENTROID PTable must exist", table); + + List<PColumn> pkColumns = table.getPKColumns(); + assertEquals(3, pkColumns.size()); + assertEquals(INDEX_NAME, pkColumns.get(0).getName().getString()); + assertEquals(PVarchar.INSTANCE, pkColumns.get(0).getDataType()); + assertFalse(pkColumns.get(0).isNullable()); + assertEquals(GENERATION_ID, pkColumns.get(1).getName().getString()); + assertEquals(PLong.INSTANCE, pkColumns.get(1).getDataType()); + assertFalse(pkColumns.get(1).isNullable()); + assertEquals(CENTROID_ID, pkColumns.get(2).getName().getString()); + assertEquals(PInteger.INSTANCE, pkColumns.get(2).getDataType()); + assertFalse(pkColumns.get(2).isNullable()); + + PColumn centroidVectorCol = table.getColumnForColumnName(CENTROID_VECTOR); + assertNotNull(centroidVectorCol); + assertEquals(PVarbinary.INSTANCE, centroidVectorCol.getDataType()); + assertTrue(centroidVectorCol.isNullable()); + } + } + + /** Tests the column names and types that JDBC DatabaseMetaData shows for the table. */ + @Test + public void testDatabaseMetaDataColumns() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + DatabaseMetaData dbmd = conn.getMetaData(); + try (ResultSet rs = + dbmd.getColumns(null, SYSTEM_CATALOG_SCHEMA, SYSTEM_VECTOR_CENTROID_TABLE, null)) { + assertTrue("Expected INDEX_NAME column", rs.next()); + assertEquals(INDEX_NAME, rs.getString(COLUMN_NAME)); + assertEquals(Types.VARCHAR, rs.getInt(DATA_TYPE)); + + assertTrue("Expected GENERATION_ID column", rs.next()); + assertEquals(GENERATION_ID, rs.getString(COLUMN_NAME)); + assertEquals(Types.BIGINT, rs.getInt(DATA_TYPE)); + + assertTrue("Expected CENTROID_ID column", rs.next()); + assertEquals(CENTROID_ID, rs.getString(COLUMN_NAME)); + assertEquals(Types.INTEGER, rs.getInt(DATA_TYPE)); + + assertTrue("Expected CENTROID_VECTOR column", rs.next()); + assertEquals(CENTROID_VECTOR, rs.getString(COLUMN_NAME)); + assertEquals(Types.VARBINARY, rs.getInt(DATA_TYPE)); + + assertFalse("No additional columns expected", rs.next()); + } + } + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java new file mode 100644 index 0000000000..074a8b1753 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorIndexIT.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DIMENSION_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE_BYTES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.coprocessor.Batch; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.BlockingRpcCallback; +import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.coprocessor.generated.MetaDataProtos; +import org.apache.phoenix.coprocessor.generated.MetaDataProtos.CreateTableRequest; +import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataResponse; +import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService; +import org.apache.phoenix.coprocessorclient.MetaDataProtocol; +import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.hbase.index.util.VersionUtil; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.protobuf.ProtobufUtil; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.types.PBson; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVectorFloat; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.ClientUtil; +import org.apache.phoenix.util.Closeables; +import org.apache.phoenix.util.IndexUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests how MetaDataEndpointImpl checks vector index metadata and writes it to SYSTEM.CATALOG. + */ +@Category(ParallelStatsDisabledTest.class) +public class VectorIndexIT extends ParallelStatsDisabledIT { + + /** Tests that the server rejects a vector index with an algorithm that it does not support. */ + @Test + public void testInvalidAlgorithmRejection() throws Exception { + String tableName = "T_INVALID_ALGO_" + generateUniqueName(); + String indexName = "IDX_INVALID_ALGO_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + try { + createVectorIndexMetadata(pconn, null, indexName, tableName, "V", PVectorFloat.INSTANCE, + 128, "UNKNOWN", "L2", 128, 16, 500); + fail("Expected vector index creation with UNKNOWN algorithm to be rejected"); + } catch (Exception e) { + assertExpectedSqlException(e, SQLExceptionCode.UNSUPPORTED_VECTOR_INDEX_ALGORITHM); + } + } + } + + /** Tests that the server rejects a vector index with a distance metric that it does not know. */ + @Test + public void testInvalidDistanceMetricRejection() throws Exception { + String tableName = "T_INVALID_METRIC_" + generateUniqueName(); + String indexName = "IDX_INVALID_METRIC_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + try { + createVectorIndexMetadata(pconn, null, indexName, tableName, "V", PVectorFloat.INSTANCE, + 128, "IVF", "MANHATTAN", 128, 16, 500); + fail("Expected vector index creation with unsupported metric to be rejected"); + } catch (Exception e) { + assertExpectedSqlException(e, SQLExceptionCode.UNSUPPORTED_VECTOR_DISTANCE_METRIC); + } + } + } + + /** Tests that the server rejects IVF metadata with a list count of zero. */ + @Test + public void testInvalidIvfParametersZeroListsRejection() throws Exception { + String tableName = "T_IVF_ZERO_LISTS_" + generateUniqueName(); + String indexName = "IDX_IVF_ZERO_LISTS_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + try { + createVectorIndexMetadata(pconn, null, indexName, tableName, "V", PVectorFloat.INSTANCE, + 128, "IVF", "L2", 128, 0, 500); + fail("Expected vector index creation with vectorIvfLists = 0 to be rejected"); + } catch (Exception e) { + assertExpectedSqlException(e, SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS); + } + } + } + + /** Tests that the server rejects IVF metadata with a sample size less than the list count. */ + @Test + public void testInvalidIvfParametersSampleSizeLessThanListsRejection() throws Exception { + String tableName = "T_IVF_SAMPLE_TOO_SMALL_" + generateUniqueName(); + String indexName = "IDX_IVF_SAMPLE_TOO_SMALL_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + try { + createVectorIndexMetadata(pconn, null, indexName, tableName, "V", PVectorFloat.INSTANCE, + 128, "IVF", "L2", 128, 32, 16); + fail("Expected vector index creation with sample_size < lists to be rejected"); + } catch (Exception e) { + assertExpectedSqlException(e, SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS); + } + } + } + + /** Tests that the server rejects vector metadata on a table that is not an index. */ + @Test + public void testVectorMetadataOnNonIndexTableRejection() throws Exception { + String tableName = "T_NOT_INDEX_" + generateUniqueName(); + String otherName = "T_OTHER_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + try { + createVectorIndexMetadata(pconn, null, otherName, tableName, "V", PVectorFloat.INSTANCE, + 128, "IVF", "L2", 128, 16, 500, PTableType.TABLE); + fail("Expected vector index metadata on a non-index table to be rejected"); + } catch (Exception e) { + assertExpectedSqlException(e, SQLExceptionCode.INVALID_VECTOR_INDEX_PARAMS); + } + } + } + + /** + * Tests that the server accepts a vector index on a VECTOR column and returns a PTable that has + * the requested vector metadata. + */ + @Test + public void testValidVectorIndexMetadataCreation() throws Exception { + String tableName = "T_VALID_" + generateUniqueName(); + String indexName = "IDX_VALID_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, V VECTOR(FLOAT, 128))"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + MetaDataResponse response = createVectorIndexMetadata(pconn, null, indexName, tableName, "V", + PVectorFloat.INSTANCE, 128, "IVF", "L2", 128, 16, 500); + assertNotNull("Expected non-null response", response); + assertEquals(MetaDataProtos.MutationCode.TABLE_NOT_FOUND, response.getReturnCode()); + assertTrue("Expected response to contain built table", response.hasTable()); + + PTable indexTable = PTableImpl.createFromProto(response.getTable()); + assertNotNull("Vector index table must exist in catalog", indexTable); + assertEquals("IVF", indexTable.getVectorIndexAlgorithm()); + assertEquals("L2", indexTable.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(128), indexTable.getVectorDimension()); + assertEquals(Integer.valueOf(16), indexTable.getVectorIvfLists()); + assertEquals(Integer.valueOf(500), indexTable.getVectorIvfSampleSize()); + } + } + + /** + * Tests that the server accepts a vector index on a BSON column and returns a PTable that has the + * requested vector metadata. + */ + @Test + public void testValidBsonVectorIndexMetadataCreation() throws Exception { + String tableName = "T_BSON_" + generateUniqueName(); + String indexName = "IDX_BSON_" + generateUniqueName(); + + try (Connection conn = DriverManager.getConnection(getUrl())) { + try (Statement stmt = conn.createStatement()) { + stmt.execute("CREATE TABLE " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, DOC BSON)"); + } + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + MetaDataResponse response = createVectorIndexMetadata(pconn, null, indexName, tableName, + "DOC", PBson.INSTANCE, null, "IVF", "COSINE", 256, 32, 1000); + assertNotNull("Expected non-null response", response); + assertEquals(MetaDataProtos.MutationCode.TABLE_NOT_FOUND, response.getReturnCode()); + assertTrue("Expected response to contain built table", response.hasTable()); + + PTable indexTable = PTableImpl.createFromProto(response.getTable()); + assertNotNull("Vector index on BSON column must exist in catalog", indexTable); + assertEquals("IVF", indexTable.getVectorIndexAlgorithm()); + assertEquals("COSINE", indexTable.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(256), indexTable.getVectorDimension()); + assertEquals(Integer.valueOf(32), indexTable.getVectorIvfLists()); + assertEquals(Integer.valueOf(1000), indexTable.getVectorIvfSampleSize()); + } + } + + private void assertExpectedSqlException(Exception e, SQLExceptionCode expectedCode) { + SQLException sqle = null; + if (e instanceof SQLException) { + sqle = (SQLException) e; + } else { + sqle = ClientUtil.parseServerException(e); + } + assertNotNull("Expected parsed SQLException from server error: " + e.getMessage(), sqle); + assertEquals("Unexpected SQL error code: " + sqle.getMessage(), expectedCode.getErrorCode(), + sqle.getErrorCode()); + assertEquals("Unexpected SQL state: " + sqle.getSQLState(), expectedCode.getSQLState(), + sqle.getSQLState()); + } + + private MetaDataResponse createVectorIndexMetadata(PhoenixConnection pconn, String schemaName, + String indexName, String baseTableName, String indexedColumnName, + PDataType<?> indexedColumnType, Integer columnSize, String algorithm, String metric, + Integer dimension, Integer lists, Integer sampleSize) throws Exception { + return createVectorIndexMetadata(pconn, schemaName, indexName, baseTableName, indexedColumnName, + indexedColumnType, columnSize, algorithm, metric, dimension, lists, sampleSize, + PTableType.INDEX); + } + + private MetaDataResponse createVectorIndexMetadata(PhoenixConnection pconn, String schemaName, + String indexName, String baseTableName, String indexedColumnName, + PDataType<?> indexedColumnType, Integer columnSize, String algorithm, String metric, + Integer dimension, Integer lists, Integer sampleSize, PTableType tableType) throws Exception { + byte[] schemaBytes = schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName); + byte[] indexTableBytes = Bytes.toBytes(indexName); + byte[] baseTableBytes = Bytes.toBytes(baseTableName); + byte[] tableKey = SchemaUtil.getTableKey(null, schemaBytes, indexTableBytes); + byte[] baseTableKey = SchemaUtil.getTableKey(null, schemaBytes, baseTableBytes); + + List<Mutation> tableMetadata = new ArrayList<>(); + + Put headerPut = new Put(tableKey); + headerPut.addColumn(TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES, + Bytes.toBytes(tableType.getSerializedValue())); + headerPut.addColumn(TABLE_FAMILY_BYTES, TABLE_SEQ_NUM_BYTES, PLong.INSTANCE.toBytes(1L)); + headerPut.addColumn(TABLE_FAMILY_BYTES, COLUMN_COUNT_BYTES, PInteger.INSTANCE.toBytes(1)); + headerPut.addColumn(TABLE_FAMILY_BYTES, INDEX_TYPE_BYTES, + new byte[] { IndexType.VECTOR_GLOBAL.getSerializedValue() }); + headerPut.addColumn(TABLE_FAMILY_BYTES, DATA_TABLE_NAME_BYTES, baseTableBytes); + + if (algorithm != null) { + headerPut.addColumn(TABLE_FAMILY_BYTES, VECTOR_INDEX_ALGORITHM_BYTES, + Bytes.toBytes(algorithm)); + } + if (metric != null) { + headerPut.addColumn(TABLE_FAMILY_BYTES, VECTOR_DISTANCE_METRIC_BYTES, Bytes.toBytes(metric)); + } + if (dimension != null) { + headerPut.addColumn(TABLE_FAMILY_BYTES, VECTOR_DIMENSION_BYTES, + PInteger.INSTANCE.toBytes(dimension)); + } + if (lists != null) { + headerPut.addColumn(TABLE_FAMILY_BYTES, VECTOR_IVF_LISTS_BYTES, + PInteger.INSTANCE.toBytes(lists)); + } + if (sampleSize != null) { + headerPut.addColumn(TABLE_FAMILY_BYTES, VECTOR_IVF_SAMPLE_SIZE_BYTES, + PInteger.INSTANCE.toBytes(sampleSize)); + } + tableMetadata.add(headerPut); + + if (indexedColumnName != null) { + String fullIndexColName = IndexUtil.getIndexColumnName(null, indexedColumnName); + byte[] colKey = SchemaUtil.getColumnKey(null, schemaName, indexName, fullIndexColName, null); + Put colPut = new Put(colKey); + if (indexedColumnType != null) { + colPut.addColumn(TABLE_FAMILY_BYTES, DATA_TYPE_BYTES, + PInteger.INSTANCE.toBytes(indexedColumnType.getSqlType())); + } + colPut.addColumn(TABLE_FAMILY_BYTES, NULLABLE_BYTES, + PInteger.INSTANCE.toBytes(ResultSetMetaData.columnNoNulls)); + colPut.addColumn(TABLE_FAMILY_BYTES, ORDINAL_POSITION_BYTES, PInteger.INSTANCE.toBytes(1)); + if (columnSize != null) { + colPut.addColumn(TABLE_FAMILY_BYTES, COLUMN_SIZE_BYTES, + PInteger.INSTANCE.toBytes(columnSize)); + } + tableMetadata.add(colPut); + } + + Put parentHeaderPut = new Put(baseTableKey); + tableMetadata.add(parentHeaderPut); + + PTable parentTable = null; + try { + parentTable = pconn.getTableNoCache(SchemaUtil.getTableName(schemaName, baseTableName)); + } catch (Exception ignored) { + } + + final List<Mutation> finalMutations = tableMetadata; + final PTable finalParentTable = parentTable; + + Table ht = pconn.getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES); + try { + Batch.Call<MetaDataService, MetaDataResponse> callable = + new Batch.Call<MetaDataService, MetaDataResponse>() { + @Override + public MetaDataResponse call(MetaDataService instance) throws IOException { + ServerRpcController controller = new ServerRpcController(); + BlockingRpcCallback<MetaDataResponse> rpcCallback = new BlockingRpcCallback<>(); + CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); + for (Mutation m : finalMutations) { + builder.addTableMetadataMutations(ProtobufUtil.toProto(m).toByteString()); + } + builder + .setClientVersion(VersionUtil.encodeVersion(MetaDataProtocol.PHOENIX_MAJOR_VERSION, + MetaDataProtocol.PHOENIX_MINOR_VERSION, MetaDataProtocol.PHOENIX_PATCH_NUMBER)); + if (finalParentTable != null) { + builder.setParentTable(PTableImpl.toProto(finalParentTable)); + } + instance.createTable(controller, builder.build(), rpcCallback); + if (controller.getFailedOn() != null) { + throw controller.getFailedOn(); + } + return rpcCallback.get(); + } + }; + + Map<byte[], MetaDataResponse> results; + try { + results = ht.coprocessorService(MetaDataService.class, tableKey, tableKey, callable); + } catch (Throwable t) { + if (t instanceof Exception) { + throw (Exception) t; + } + throw new Exception(t); + } + assertNotNull("Expected non-null coprocessor response", results); + return results.values().iterator().next(); + } finally { + Closeables.closeQuietly(ht); + } + } +} diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorSystemCatalogIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorSystemCatalogIT.java new file mode 100644 index 0000000000..4e600a0d32 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VectorSystemCatalogIT.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_CENTROID_GENERATION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DIMENSION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_DISTANCE_METRIC; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_INDEX_ALGORITHM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_LISTS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VECTOR_IVF_SAMPLE_SIZE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVarchar; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests that the vector index metadata columns of SYSTEM.CATALOG are the same in the raw catalog + * rows, in JDBC DatabaseMetaData, and in the PTable. + */ +@Category(ParallelStatsDisabledTest.class) +public class VectorSystemCatalogIT extends ParallelStatsDisabledIT { + + private static final List<String> VECTOR_METADATA_COLUMNS = + Arrays.asList(VECTOR_INDEX_ALGORITHM, VECTOR_DISTANCE_METRIC, VECTOR_DIMENSION, + VECTOR_IVF_LISTS, VECTOR_IVF_SAMPLE_SIZE, VECTOR_CENTROID_GENERATION); + + private static final Map<String, PDataType<?>> EXPECTED_COLUMN_TYPES = new HashMap<>(); + static { + EXPECTED_COLUMN_TYPES.put(VECTOR_INDEX_ALGORITHM, PVarchar.INSTANCE); + EXPECTED_COLUMN_TYPES.put(VECTOR_DISTANCE_METRIC, PVarchar.INSTANCE); + EXPECTED_COLUMN_TYPES.put(VECTOR_DIMENSION, PInteger.INSTANCE); + EXPECTED_COLUMN_TYPES.put(VECTOR_IVF_LISTS, PInteger.INSTANCE); + EXPECTED_COLUMN_TYPES.put(VECTOR_IVF_SAMPLE_SIZE, PInteger.INSTANCE); + EXPECTED_COLUMN_TYPES.put(VECTOR_CENTROID_GENERATION, PLong.INSTANCE); + } + + /** + * Tests that each vector metadata column exists and has the correct SQL type in the + * SYSTEM.CATALOG rows, DatabaseMetaData, and the PTable. The test also checks that the column is + * nullable in the SYSTEM.CATALOG rows and the PTable. + */ + @Test + public void testAllVectorIndexMetadataColumnsExistWithCorrectSchema() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl())) { + PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); + + // Read the raw SYSTEM.CATALOG rows + String query = "SELECT " + COLUMN_NAME + ", " + DATA_TYPE + ", " + NULLABLE + " FROM " + + SYSTEM_CATALOG_NAME + " WHERE " + TABLE_NAME + " = ? AND " + COLUMN_NAME + " = ?"; + for (String colName : VECTOR_METADATA_COLUMNS) { + try (PreparedStatement stmt = conn.prepareStatement(query)) { + stmt.setString(1, SYSTEM_CATALOG_TABLE); + stmt.setString(2, colName); + try (ResultSet rs = stmt.executeQuery()) { + assertTrue("Column " + colName + " must exist in SYSTEM.CATALOG", rs.next()); + assertEquals(colName, rs.getString(COLUMN_NAME)); + PDataType<?> expectedType = EXPECTED_COLUMN_TYPES.get(colName); + assertEquals("Data type mismatch for " + colName, expectedType.getSqlType(), + rs.getInt(DATA_TYPE)); + assertEquals("Column " + colName + " should be nullable", + DatabaseMetaData.columnNullable, rs.getInt(NULLABLE)); + } + } + } + + // Read the JDBC DatabaseMetaData column definitions + DatabaseMetaData dbmd = conn.getMetaData(); + for (String colName : VECTOR_METADATA_COLUMNS) { + try (ResultSet rs = + dbmd.getColumns(null, SYSTEM_CATALOG_SCHEMA, SYSTEM_CATALOG_TABLE, colName)) { + assertTrue("DatabaseMetaData must return column " + colName, rs.next()); + assertEquals(colName, rs.getString("COLUMN_NAME")); + PDataType<?> expectedType = EXPECTED_COLUMN_TYPES.get(colName); + assertEquals(expectedType.getSqlType(), rs.getInt("DATA_TYPE")); + } + } + + // Read the column definitions of a PTable that is not from the cache + PTable syscatTable = pconn.getTableNoCache(SYSTEM_CATALOG_NAME); + assertNotNull("SYSTEM.CATALOG PTable must exist", syscatTable); + for (String colName : VECTOR_METADATA_COLUMNS) { + PColumn pcol = syscatTable.getColumnForColumnName(colName); + assertNotNull("PColumn for " + colName + " must exist in PTable", pcol); + assertEquals(EXPECTED_COLUMN_TYPES.get(colName), pcol.getDataType()); + assertTrue("Column " + colName + " must be nullable in PTable", pcol.isNullable()); + } + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java new file mode 100644 index 0000000000..79b4007cd3 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/VectorIndexTypeTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Collections; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.coprocessor.generated.PTableProtos; +import org.apache.phoenix.schema.PTable.IndexType; +import org.junit.Test; + +/** + * Tests the {@link PTable.IndexType#VECTOR_GLOBAL} index type and the vector metadata of a PTable. + * The tests cover serialized values, SQL tokens, protobuf round trips, and accessors. + */ +public class VectorIndexTypeTest { + + @Test + public void testFromSerializedValueVectorGlobal() { + IndexType indexType = IndexType.fromSerializedValue((byte) 4); + assertEquals(IndexType.VECTOR_GLOBAL, indexType); + } + + @Test + public void testVectorGlobalRoundTrip() { + assertEquals((byte) 4, IndexType.VECTOR_GLOBAL.getSerializedValue()); + assertEquals(IndexType.VECTOR_GLOBAL, + IndexType.fromSerializedValue(IndexType.VECTOR_GLOBAL.getSerializedValue())); + } + + @Test + public void testAllIndexTypesRoundTrip() { + for (IndexType type : IndexType.values()) { + assertEquals(type, IndexType.fromSerializedValue(type.getSerializedValue())); + } + } + + @Test + public void testLegacyRejectionUnknownSerializedValue() { + try { + IndexType.fromSerializedValue((byte) 99); + fail("Expected IllegalArgumentException for unknown serialized value 99"); + } catch (IllegalArgumentException e) { + assertTrue("Exception message should mention invalid value", e.getMessage().contains("99")); + } + } + + @Test + public void testBoundaryAndInvalidValuesRejection() { + try { + IndexType.fromSerializedValue((byte) 0); + fail("Expected IllegalArgumentException for invalid serialized value 0"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("0")); + } + + try { + IndexType.fromSerializedValue((byte) -1); + fail("Expected IllegalArgumentException for invalid serialized value -1"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("-1")); + } + + try { + IndexType.fromSerializedValue((byte) (IndexType.values().length + 1)); + fail("Expected IllegalArgumentException for out-of-bounds serialized value"); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void testFromToken() { + assertEquals(IndexType.VECTOR_GLOBAL, IndexType.fromToken("VECTOR_GLOBAL")); + assertEquals(IndexType.VECTOR_GLOBAL, IndexType.fromToken(" vector_global ")); + assertEquals(IndexType.GLOBAL, IndexType.fromToken("GLOBAL")); + assertEquals(IndexType.LOCAL, IndexType.fromToken("LOCAL")); + assertEquals(IndexType.UNCOVERED_GLOBAL, IndexType.fromToken("UNCOVERED_GLOBAL")); + } + + @Test + public void testGetBytes() { + assertArrayEquals(Bytes.toBytes("VECTOR_GLOBAL"), IndexType.VECTOR_GLOBAL.getBytes()); + assertArrayEquals(Bytes.toBytes("GLOBAL"), IndexType.GLOBAL.getBytes()); + assertArrayEquals(Bytes.toBytes("LOCAL"), IndexType.LOCAL.getBytes()); + assertArrayEquals(Bytes.toBytes("UNCOVERED_GLOBAL"), IndexType.UNCOVERED_GLOBAL.getBytes()); + } + + @Test + public void testPTableImplSerializationRoundTripWithVectorGlobal() throws Exception { + PTable table = new PTableImpl.Builder().setType(PTableType.INDEX) + .setIndexType(IndexType.VECTOR_GLOBAL).setName(PNameFactory.newName("IDX_VEC_GLOBAL")) + .setTableName(PNameFactory.newName("IDX_VEC_GLOBAL")) + .setParentTableName(PNameFactory.newName("DATA_TBL")).setAllColumns(Collections.emptyList()) + .setPkColumns(Collections.emptyList()).setIndexes(Collections.emptyList()) + .setPhysicalNames(Collections.emptyList()).setVectorIndexAlgorithm("IVF") + .setVectorDistanceMetric("COSINE").setVectorDimension(128).setVectorIvfLists(64) + .setVectorIvfSampleSize(2048).setVectorCentroidGeneration(101L).build(); + + assertEquals(IndexType.VECTOR_GLOBAL, table.getIndexType()); + + PTableProtos.PTable proto = PTableImpl.toProto(table); + assertNotNull(proto); + assertTrue(proto.hasIndexType()); + assertEquals((byte) 4, proto.getIndexType().toByteArray()[0]); + + PTable deserialized = PTableImpl.createFromProto(proto); + assertNotNull(deserialized); + assertEquals(IndexType.VECTOR_GLOBAL, deserialized.getIndexType()); + assertEquals("IVF", deserialized.getVectorIndexAlgorithm()); + } + + @Test + public void testDelegateTableWithVectorGlobal() throws Exception { + PTable inner = new PTableImpl.Builder().setIndexType(IndexType.VECTOR_GLOBAL).build(); + + DelegateTable delegate = new DelegateTable(inner); + assertEquals(IndexType.VECTOR_GLOBAL, delegate.getIndexType()); + } + + @Test + public void testPTableIsVectorIndexPredicate() throws Exception { + PTable vectorTable = new PTableImpl.Builder().setIndexType(IndexType.VECTOR_GLOBAL) + .setVectorIndexAlgorithm("IVF").build(); + assertTrue("Expected isVectorIndex to be true", vectorTable.isVectorIndex()); + assertEquals("IVF", vectorTable.getVectorIndexAlgorithm()); + + PTable nonVectorTable = new PTableImpl.Builder().build(); + assertFalse("Expected isVectorIndex to be false", nonVectorTable.isVectorIndex()); + assertNull(nonVectorTable.getVectorIndexAlgorithm()); + } + + @Test + public void testPTableVectorMetadataAccessors() throws Exception { + PTable table = + new PTableImpl.Builder().setIndexType(IndexType.VECTOR_GLOBAL).setVectorIndexAlgorithm("IVF") + .setVectorDistanceMetric("COSINE").setVectorDimension(128).setVectorIvfLists(64) + .setVectorIvfSampleSize(2048).setVectorCentroidGeneration(1001L).build(); + + assertTrue(table.isVectorIndex()); + assertEquals("IVF", table.getVectorIndexAlgorithm()); + assertEquals("COSINE", table.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(128), table.getVectorDimension()); + assertEquals(Integer.valueOf(64), table.getVectorIvfLists()); + assertEquals(Integer.valueOf(2048), table.getVectorIvfSampleSize()); + assertEquals(Long.valueOf(1001L), table.getVectorCentroidGeneration()); + } + + @Test + public void testPTableSerializationRoundTrip() throws Exception { + PTable original = new PTableImpl.Builder().setType(PTableType.INDEX) + .setName(PNameFactory.newName("IDX_VEC")).setTableName(PNameFactory.newName("IDX_VEC")) + .setIndexType(IndexType.VECTOR_GLOBAL).setParentTableName(PNameFactory.newName("DATA_TBL")) + .setAllColumns(Collections.emptyList()).setPkColumns(Collections.emptyList()) + .setIndexes(Collections.emptyList()).setPhysicalNames(Collections.emptyList()) + .setVectorIndexAlgorithm("IVF").setVectorDistanceMetric("L2").setVectorDimension(256) + .setVectorIvfLists(32).setVectorIvfSampleSize(1024).setVectorCentroidGeneration(42L).build(); + + PTableProtos.PTable proto = PTableImpl.toProto(original); + assertNotNull(proto); + assertTrue(proto.hasVectorIndexAlgorithm()); + assertEquals("IVF", proto.getVectorIndexAlgorithm()); + assertTrue(proto.hasVectorDistanceMetric()); + assertEquals("L2", proto.getVectorDistanceMetric()); + assertTrue(proto.hasVectorDimension()); + assertEquals(256, proto.getVectorDimension()); + assertTrue(proto.hasVectorIvfLists()); + assertEquals(32, proto.getVectorIvfLists()); + assertTrue(proto.hasVectorIvfSampleSize()); + assertEquals(1024, proto.getVectorIvfSampleSize()); + assertTrue(proto.hasVectorCentroidGeneration()); + assertEquals(42L, proto.getVectorCentroidGeneration()); + + PTable deserialized = PTableImpl.createFromProto(proto); + assertNotNull(deserialized); + assertTrue(deserialized.isVectorIndex()); + assertEquals("IVF", deserialized.getVectorIndexAlgorithm()); + assertEquals("L2", deserialized.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(256), deserialized.getVectorDimension()); + assertEquals(Integer.valueOf(32), deserialized.getVectorIvfLists()); + assertEquals(Integer.valueOf(1024), deserialized.getVectorIvfSampleSize()); + assertEquals(Long.valueOf(42L), deserialized.getVectorCentroidGeneration()); + } + + @Test + public void testPTableBuilderFromExisting() throws Exception { + PTable original = new PTableImpl.Builder().setType(PTableType.INDEX) + .setName(PNameFactory.newName("IDX_VEC")).setTableName(PNameFactory.newName("IDX_VEC")) + .setIndexType(IndexType.VECTOR_GLOBAL).setParentTableName(PNameFactory.newName("DATA_TBL")) + .setPhysicalNames(Collections.emptyList()).setVectorIndexAlgorithm("DISKANN") + .setVectorDistanceMetric("INNER_PRODUCT").setVectorDimension(512).setVectorIvfLists(128) + .setVectorIvfSampleSize(4096).setVectorCentroidGeneration(777L).build(); + + PTable cloned = PTableImpl.builderFromExisting(original).build(); + assertTrue(cloned.isVectorIndex()); + assertEquals("DISKANN", cloned.getVectorIndexAlgorithm()); + assertEquals("INNER_PRODUCT", cloned.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(512), cloned.getVectorDimension()); + assertEquals(Integer.valueOf(128), cloned.getVectorIvfLists()); + assertEquals(Integer.valueOf(4096), cloned.getVectorIvfSampleSize()); + assertEquals(Long.valueOf(777L), cloned.getVectorCentroidGeneration()); + } + + @Test + public void testDelegateTableVectorMetadata() throws Exception { + PTable inner = new PTableImpl.Builder().setIndexType(IndexType.VECTOR_GLOBAL) + .setVectorIndexAlgorithm("IVF").setVectorDistanceMetric("COSINE").setVectorDimension(64) + .setVectorIvfLists(16).setVectorIvfSampleSize(512).setVectorCentroidGeneration(99L).build(); + + DelegateTable delegate = new DelegateTable(inner); + assertTrue(delegate.isVectorIndex()); + assertEquals("IVF", delegate.getVectorIndexAlgorithm()); + assertEquals("COSINE", delegate.getVectorDistanceMetric()); + assertEquals(Integer.valueOf(64), delegate.getVectorDimension()); + assertEquals(Integer.valueOf(16), delegate.getVectorIvfLists()); + assertEquals(Integer.valueOf(512), delegate.getVectorIvfSampleSize()); + assertEquals(Long.valueOf(99L), delegate.getVectorCentroidGeneration()); + } + + @Test + public void testPTableSerializationRoundTripWithoutVectorFields() throws Exception { + PTable original = new PTableImpl.Builder().setType(PTableType.TABLE) + .setName(PNameFactory.newName("NON_VECTOR")).setTableName(PNameFactory.newName("NON_VECTOR")) + .setAllColumns(Collections.emptyList()).setPkColumns(Collections.emptyList()) + .setIndexes(Collections.emptyList()).setPhysicalNames(Collections.emptyList()).build(); + + assertFalse("Non-vector table must not be a vector index", original.isVectorIndex()); + + PTableProtos.PTable proto = PTableImpl.toProto(original); + assertNotNull(proto); + assertFalse("Proto should not have vectorIndexAlgorithm", proto.hasVectorIndexAlgorithm()); + assertFalse("Proto should not have vectorDistanceMetric", proto.hasVectorDistanceMetric()); + assertFalse("Proto should not have vectorDimension", proto.hasVectorDimension()); + assertFalse("Proto should not have vectorIvfLists", proto.hasVectorIvfLists()); + assertFalse("Proto should not have vectorIvfSampleSize", proto.hasVectorIvfSampleSize()); + assertFalse("Proto should not have vectorCentroidGeneration", + proto.hasVectorCentroidGeneration()); + + PTable deserialized = PTableImpl.createFromProto(proto); + assertNotNull(deserialized); + assertFalse("Deserialized non-vector table must not be a vector index", + deserialized.isVectorIndex()); + assertNull("vectorIndexAlgorithm must be null", deserialized.getVectorIndexAlgorithm()); + assertNull("vectorDistanceMetric must be null", deserialized.getVectorDistanceMetric()); + assertNull("vectorDimension must be null", deserialized.getVectorDimension()); + assertNull("vectorIvfLists must be null", deserialized.getVectorIvfLists()); + assertNull("vectorIvfSampleSize must be null", deserialized.getVectorIvfSampleSize()); + assertNull("vectorCentroidGeneration must be null", deserialized.getVectorCentroidGeneration()); + } +}
