This is an automated email from the ASF dual-hosted git repository. github-actions[bot] pushed a commit to branch cherry-pick-d4d95ebf-to-branch-1.3 in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 4154b5f30eef03865526fb95cc175844c959a9f0 Author: Yuhui <[email protected]> AuthorDate: Fri Oct 23 15:23:25 2026 +0800 [#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067) ### What changes were proposed in this pull request? Map StarRocks `SHOW INDEX` `Index_type` to Gravitino index types when loading a table: BITMAP to `BITMAP`, GIN to `INVERTED`, VECTOR to `VECTOR`. Indexes without a Gravitino equivalent (e.g. NGRAMBF) are skipped. Update the StarRocks catalog docs accordingly. ### Why are the changes needed? Every index returned by StarRocks `SHOW INDEX` (BITMAP, NGRAMBF, GIN, ...) was reported as a `PRIMARY_KEY` index, so readers saw primary keys that the table does not have, while the table's real keys are not returned by `SHOW INDEX` at all. Fix: #14066 ### Does this PR introduce _any_ user-facing change? Yes. Loaded StarRocks tables report BITMAP, GIN and VECTOR indexes with their real types, and no longer report NGRAMBF indexes as primary keys. ### How was this patch tested? Added a unit test in `TestStarRocksTableOperationsSqlGeneration`, and verified BITMAP, GIN and NGRAMBF against StarRocks 3.3 manually. --------- Co-authored-by: Jerry Shao <[email protected]> # Conflicts: # catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java # clients/client-python/gravitino/api/rel/indexes/index.py # clients/client-python/tests/unittests/dto/rel/test_index_serdes.py --- .../org/apache/gravitino/rel/indexes/Index.java | 9 +- .../operations/StarRocksTableOperations.java | 45 +++++++++- .../operation/TestStarRocksTableOperations.java | 25 ++++++ .../TestStarRocksTableOperationsSqlGeneration.java | 90 ++++++++++++++++++++ .../gravitino/api/rel/indexes/index.py | 21 +++++ .../tests/unittests/dto/rel/test_index_serdes.py | 95 ++++++++++++++++++++++ docs/jdbc-starrocks-catalog.md | 10 ++- 7 files changed, 289 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java index 9f725c19cc..173f87c590 100644 --- a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java +++ b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java @@ -92,7 +92,8 @@ public interface Index { /** * Bitmap index is a type of database index that uses bit arrays (bitmaps) to represent the * presence or absence of values in a column, enabling efficient querying and data retrieval. - * Currently, this type is only applicable to Lance. + * Currently, this type is applicable to Lance, is accepted when creating Doris tables, and is + * reported when loading StarRocks tables. */ BITMAP, /** @@ -103,12 +104,14 @@ public interface Index { /** * INVERTED index is a data structure used to optimize full-text searches by mapping terms to * their locations within a dataset, allowing for quick retrieval of documents containing - * specific words or phrases. Currently, this type is only applicable to Lance. + * specific words or phrases. Currently, this type is applicable to Lance and Doris, and is + * reported when loading StarRocks tables. */ INVERTED, /** * VECTOR index is used to optimize similarity searches in high-dimensional vector spaces. - * Currently, this type is only applicable to Lance. + * Currently, this type is applicable to Lance and Doris, and is reported when loading StarRocks + * tables. */ VECTOR, /** IVF_FLAT (Inverted File with Flat quantization) is an indexing method used for efficient */ diff --git a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java index 8d7b39ca2e..d9f0fe1650 100644 --- a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java +++ b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java @@ -32,10 +32,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -253,12 +255,24 @@ public class StarRocksTableOperations extends JdbcTableOperations { try (PreparedStatement preparedStatement = connection.prepareStatement(sql); ResultSet resultSet = preparedStatement.executeQuery()) { + // SHOW INDEX lists only secondary indexes, one row per single-column index; table keys are + // not included. List<Index> indexes = new ArrayList<>(); while (resultSet.next()) { String indexName = resultSet.getString("Key_name"); String columnName = resultSet.getString("Column_name"); - indexes.add( - Indexes.of(Index.IndexType.PRIMARY_KEY, indexName, new String[][] {{columnName}})); + String starRocksIndexType = resultSet.getString("Index_type"); + Index.IndexType indexType = mapStarRocksIndexType(starRocksIndexType); + if (indexType == null) { + LOG.warn( + "Skip StarRocks index '{}' on table {}.{} with unsupported type '{}'", + indexName, + databaseName, + tableName, + starRocksIndexType); + continue; + } + indexes.add(Indexes.of(indexType, indexName, new String[][] {{columnName}})); } return indexes; } catch (SQLException e) { @@ -614,4 +628,31 @@ public class StarRocksTableOperations extends JdbcTableOperations { .build(); return appendColumnDefinition(newColumn, sqlBuilder).toString(); } + + /** + * Maps the {@code Index_type} value of StarRocks {@code SHOW INDEX} to a Gravitino index type. + * The value may carry index properties, e.g. {@code GIN("parser" = "english")}; only the type + * name is used and the properties are not kept, since StarRocks indexes cannot be created or + * altered through Gravitino. + * + * @param indexType the {@code Index_type} value returned by StarRocks + * @return the Gravitino index type, or null if the type has no Gravitino equivalent + */ + @Nullable + private static Index.IndexType mapStarRocksIndexType(@Nullable String indexType) { + if (indexType == null) { + return null; + } + String typeName = StringUtils.substringBefore(indexType, "(").trim(); + switch (typeName.toUpperCase(Locale.ROOT)) { + case "BITMAP": + return Index.IndexType.BITMAP; + case "GIN": + return Index.IndexType.INVERTED; + case "VECTOR": + return Index.IndexType.VECTOR; + default: + return null; + } + } } diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java index 39763b3574..a70e10d553 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java @@ -776,4 +776,29 @@ public class TestStarRocksTableOperations extends TestStarRocks { TABLE_OPERATIONS.drop(databaseName, tableName); } } + + @ParameterizedTest + @ValueSource(strings = {"PRIMARY", "UNIQUE", "DUPLICATE"}) + void testLoadTableReportsOnlySecondaryIndexes(String keyType) throws SQLException { + String tableName = GravitinoITUtils.genRandomName("starrocks_index"); + try (Connection connection = DATA_SOURCE.getConnection(); + Statement statement = connection.createStatement()) { + statement.executeUpdate( + String.format( + "CREATE TABLE `%s`.`%s` (`id` INT NOT NULL, `region` VARCHAR(32) NOT NULL, " + + "INDEX `idx_region` (`region`) USING BITMAP) " + + "%s KEY(`id`, `region`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 " + + "PROPERTIES (\"replication_num\" = \"1\")", + databaseName, tableName, keyType)); + } + try { + JdbcTable table = TABLE_OPERATIONS.load(databaseName, tableName); + Assertions.assertEquals(1, table.index().length); + Assertions.assertEquals(Index.IndexType.BITMAP, table.index()[0].type()); + Assertions.assertEquals("idx_region", table.index()[0].name()); + Assertions.assertArrayEquals(new String[][] {{"region"}}, table.index()[0].fieldNames()); + } finally { + TABLE_OPERATIONS.drop(databaseName, tableName); + } + } } diff --git a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java index d2665ca33c..a2a128dacd 100644 --- a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java +++ b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java @@ -19,6 +19,13 @@ package org.apache.gravitino.catalog.starrocks.operation; import java.sql.Connection; +<<<<<<< HEAD +======= +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +>>>>>>> d4d95ebfe ([#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067)) import java.util.Collections; import java.util.List; import org.apache.gravitino.StringIdentifier; @@ -45,6 +52,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EmptySource; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.NullSource; +import org.mockito.Mockito; public class TestStarRocksTableOperationsSqlGeneration { @@ -91,6 +99,11 @@ public class TestStarRocksTableOperationsSqlGeneration { return generateAlterTableSql("database", tableName, changes); } + List<Index> indexes(Connection connection, String databaseName, String tableName) + throws SQLException { + return getIndexes(connection, databaseName, tableName); + } + @Override protected JdbcTable getOrCreateTable( String databaseName, String tableName, JdbcTable lazyLoadCreateTable) { @@ -375,4 +388,81 @@ public class TestStarRocksTableOperationsSqlGeneration { Assertions.assertEquals( "StarRocks does not support adding an auto-increment column: col2", exception.getMessage()); } +<<<<<<< HEAD +======= + + @Test + void testGetIndexesMapsTypesAndSkipsUnsupported() throws SQLException { + TestableStarRocksTableOperations ops = new TestableStarRocksTableOperations(); + Connection connection = Mockito.mock(Connection.class); + PreparedStatement statement = Mockito.mock(PreparedStatement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + Mockito.when(connection.prepareStatement("SHOW INDEX FROM `table` FROM `database`")) + .thenReturn(statement); + Mockito.when(statement.executeQuery()).thenReturn(resultSet); + Mockito.when(resultSet.next()).thenReturn(true, true, true, true, true, true, false); + Mockito.when(resultSet.getString("Key_name")) + .thenReturn("idx_region", "idx_ngram", "idx_text", "idx_vector", "idx_unknown", "idx_null"); + Mockito.when(resultSet.getString("Column_name")) + .thenReturn("region", "name", "text", "embedding", "c5", "c6"); + Mockito.when(resultSet.getString("Index_type")) + .thenReturn( + "BITMAP", + "NGRAMBF(\"gram_num\" = \"4\")", + "GIN(\"parser\" = \"english\")", + "VECTOR(\"index_type\" = \"hnsw\", \"dim\" = \"3\")", + "UNKNOWN", + null); + + List<Index> indexes = ops.indexes(connection, "database", "table"); + + Assertions.assertEquals(3, indexes.size()); + Assertions.assertEquals(Index.IndexType.BITMAP, indexes.get(0).type()); + Assertions.assertEquals("idx_region", indexes.get(0).name()); + Assertions.assertArrayEquals(new String[][] {{"region"}}, indexes.get(0).fieldNames()); + Assertions.assertEquals(Index.IndexType.INVERTED, indexes.get(1).type()); + Assertions.assertEquals("idx_text", indexes.get(1).name()); + Assertions.assertArrayEquals(new String[][] {{"text"}}, indexes.get(1).fieldNames()); + Assertions.assertEquals(Index.IndexType.VECTOR, indexes.get(2).type()); + Assertions.assertEquals("idx_vector", indexes.get(2).name()); + Assertions.assertArrayEquals(new String[][] {{"embedding"}}, indexes.get(2).fieldNames()); + } + + @Test + void testRenameColumnQuotesIdentifiers() { + TestableStarRocksTableOperations ops = new TestableStarRocksTableOperations("id", "order-id"); + + String alterSql = + ops.alterTableSql("test_table", TableChange.renameColumn(new String[] {"order-id"}, "key")); + + Assertions.assertEquals( + "ALTER TABLE `test_table`\nRENAME COLUMN `order-id` TO `key`;", alterSql); + } + + @Test + void testDeleteMissingColumnIfExistsSkipsAlter() { + ConnectionCountingDataSource connections = new ConnectionCountingDataSource(); + TestableStarRocksTableOperations ops = + new TestableStarRocksTableOperations(connections, "id", "col1"); + TableChange deleteMissing = TableChange.deleteColumn(new String[] {"missing"}, true); + + Assertions.assertEquals("", ops.alterTableSql("test_table", deleteMissing)); + + ops.alterTable("database", "test_table", deleteMissing); + Assertions.assertTrue(connections.executedSql().isEmpty()); + } + + @Test + void testDeleteMissingColumnIfExistsWithOtherChanges() { + TestableStarRocksTableOperations ops = new TestableStarRocksTableOperations("id", "col1"); + + String alterSql = + ops.alterTableSql( + "test_table", + TableChange.deleteColumn(new String[] {"missing"}, true), + TableChange.deleteColumn(new String[] {"col1"}, false)); + + Assertions.assertEquals("ALTER TABLE `test_table`\nDROP COLUMN `col1`;", alterSql); + } +>>>>>>> d4d95ebfe ([#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067)) } diff --git a/clients/client-python/gravitino/api/rel/indexes/index.py b/clients/client-python/gravitino/api/rel/indexes/index.py index 777e3b4813..e22509837f 100644 --- a/clients/client-python/gravitino/api/rel/indexes/index.py +++ b/clients/client-python/gravitino/api/rel/indexes/index.py @@ -50,6 +50,27 @@ class Index(ABC): columns that are not designated as the primary key. The uniqueness constraint imposed by UNIQUE KEY helps in avoiding redundancy and ensuring data accuracy in the database.""" +<<<<<<< HEAD +======= + BITMAP = "BITMAP" + """Bitmap index, which uses bit arrays to represent the presence of values in a column.""" + + INVERTED = "INVERTED" + """Inverted index, which maps terms to their locations to speed up full-text searches.""" + + VECTOR = "VECTOR" + """Vector index, which speeds up similarity searches in high-dimensional vector spaces.""" + + DATA_SKIPPING_ANNOY = "DATA_SKIPPING_ANNOY" + """Legacy ClickHouse Annoy index metadata. This type is metadata-only.""" + + DATA_SKIPPING_USEARCH = "DATA_SKIPPING_USEARCH" + """Legacy ClickHouse USearch index metadata. This type is metadata-only.""" + + DATA_SKIPPING_VECTOR_SIMILARITY = "DATA_SKIPPING_VECTOR_SIMILARITY" + """ClickHouse vector similarity data skipping index.""" + +>>>>>>> d4d95ebfe ([#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067)) @abstractmethod def type(self) -> IndexType: """Returns the type of the index. eg: `PRIMARY_KEY` and `UNIQUE_KEY`. diff --git a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py index 790f102a24..73639de062 100644 --- a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py +++ b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py @@ -116,3 +116,98 @@ class TestIndexSerdes(unittest.TestCase): json_dict = json.loads(json_string) serialized_dict = json.loads(mock_data_class.to_json()) self.assertDictEqual(json_dict, serialized_dict) +<<<<<<< HEAD +======= + + def test_index_serdes_legacy_clickhouse_metadata(self): + annoy_properties = { + "annoy_trees": "100", + "clickhouse_type_full": "annoy(100)", + "granularity": "1", + } + usearch_properties = { + "clickhouse_type_full": "usearch('cosineDistance')", + "granularity": "1", + "usearch_distance_function": "cosineDistance", + } + + annoy = IndexSerdes.deserialize( + { + "indexType": "data_skipping_annoy", + "name": "idx_annoy", + "fieldNames": [["embedding"]], + "properties": annoy_properties, + } + ) + usearch = IndexSerdes.deserialize( + { + "indexType": "data_skipping_usearch", + "name": "idx_usearch", + "fieldNames": [["embedding"]], + "properties": usearch_properties, + } + ) + + self.assertEqual(Index.IndexType.DATA_SKIPPING_ANNOY, annoy.type()) + self.assertEqual(annoy_properties, annoy.properties()) + self.assertEqual(Index.IndexType.DATA_SKIPPING_USEARCH, usearch.type()) + self.assertEqual(usearch_properties, usearch.properties()) + + serialized_json = MockDataClass([annoy, usearch]).to_json() + serialized = json.loads(serialized_json) + self.assertEqual(annoy_properties, serialized["indexes"][0]["properties"]) + self.assertEqual(usearch_properties, serialized["indexes"][1]["properties"]) + + round_tripped = MockDataClass.from_json(serialized_json).indexes + self.assertEqual(Index.IndexType.DATA_SKIPPING_ANNOY, round_tripped[0].type()) + self.assertEqual(annoy_properties, round_tripped[0].properties()) + self.assertEqual(Index.IndexType.DATA_SKIPPING_USEARCH, round_tripped[1].type()) + self.assertEqual(usearch_properties, round_tripped[1].properties()) + + def test_index_serdes_bitmap_inverted_vector(self): + for index_type in ( + Index.IndexType.BITMAP, + Index.IndexType.INVERTED, + Index.IndexType.VECTOR, + ): + index = IndexSerdes.deserialize( + { + "indexType": index_type.value, + "name": "idx", + "fieldNames": [["col"]], + } + ) + self.assertEqual(index_type, index.type()) + self.assertEqual( + index_type.value, IndexSerdes.serialize(index)["indexType"] + ) + + def test_vector_similarity_index_properties_round_trip(self): + json_string = """ + { + "indexes": [ + { + "indexType": "DATA_SKIPPING_VECTOR_SIMILARITY", + "name": "idx_vector", + "fieldNames": [["embedding"]], + "properties": { + "type": "hnsw", + "distance_function": "L2Distance", + "dimensions": "3" + } + } + ] + } + """ + + index = MockDataClass.from_json(json_string).indexes[0] + self.assertEqual(index.type(), Index.IndexType.DATA_SKIPPING_VECTOR_SIMILARITY) + self.assertEqual( + index.properties(), + {"type": "hnsw", "distance_function": "L2Distance", "dimensions": "3"}, + ) + self.assertDictEqual( + json.loads(json_string), + json.loads(MockDataClass(indexes=[index]).to_json()), + ) +>>>>>>> d4d95ebfe ([#14066] fix(starrocks): Map index types when loading StarRocks tables (#14067)) diff --git a/docs/jdbc-starrocks-catalog.md b/docs/jdbc-starrocks-catalog.md index ad2d9850ce..6b41d8ac30 100644 --- a/docs/jdbc-starrocks-catalog.md +++ b/docs/jdbc-starrocks-catalog.md @@ -149,7 +149,15 @@ Unsupported for now. ### Table Indexes -Unsupported. Creating a table with non-empty index definitions is rejected before the table is created. +Creating a table with non-empty index definitions is rejected before the table is created. + +When loading a table, Gravitino reports StarRocks secondary indexes with the following type mapping. Indexes of other types (e.g. `NGRAMBF`) are not reported, and table keys are not reported as indexes. + +| StarRocks Index Type | Gravitino Index Type | +|----------------------|----------------------| +| `BITMAP` | `BITMAP` | +| `GIN` | `INVERTED` | +| `VECTOR` | `VECTOR` | ### Table Partitioning
