This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new d4d95ebfe5 [#14066] fix(starrocks): Map index types when loading
StarRocks tables (#14067)
d4d95ebfe5 is described below
commit d4d95ebfe5c92e18e2ae2e8ae88d95aeb0c0251d
Author: Yuhui <[email protected]>
AuthorDate: Fri Oct 23 15:23:25 2026 +0800
[#14066] fix(starrocks): Map index types when loading StarRocks tables
(#14067)
### What changes were proposed in this pull request?
Map StarRocks `SHOW INDEX` `Index_type` to Gravitino index types when
loading a table: BITMAP to `BITMAP`, GIN to `INVERTED`, VECTOR to
`VECTOR`. Indexes without a Gravitino equivalent (e.g. NGRAMBF) are
skipped. Update the StarRocks catalog docs accordingly.
### Why are the changes needed?
Every index returned by StarRocks `SHOW INDEX` (BITMAP, NGRAMBF, GIN,
...) was reported as a `PRIMARY_KEY` index, so readers saw primary keys
that the table does not have, while the table's real keys are not
returned by `SHOW INDEX` at all.
Fix: #14066
### Does this PR introduce _any_ user-facing change?
Yes. Loaded StarRocks tables report BITMAP, GIN and VECTOR indexes with
their real types, and no longer report NGRAMBF indexes as primary keys.
### How was this patch tested?
Added a unit test in `TestStarRocksTableOperationsSqlGeneration`, and
verified BITMAP, GIN and NGRAMBF against StarRocks 3.3 manually.
---------
Co-authored-by: Jerry Shao <[email protected]>
---
.../org/apache/gravitino/rel/indexes/Index.java | 9 +++--
.../operations/StarRocksTableOperations.java | 45 ++++++++++++++++++++-
.../operation/TestStarRocksTableOperations.java | 25 ++++++++++++
.../TestStarRocksTableOperationsSqlGeneration.java | 46 ++++++++++++++++++++++
.../gravitino/api/rel/indexes/index.py | 9 +++++
.../tests/unittests/dto/rel/test_index_serdes.py | 18 +++++++++
docs/jdbc-starrocks-catalog.md | 10 ++++-
7 files changed, 156 insertions(+), 6 deletions(-)
diff --git a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
index dbfdbe0195..e7cf8b429c 100644
--- a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
+++ b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
@@ -92,7 +92,8 @@ public interface Index {
/**
* Bitmap index is a type of database index that uses bit arrays (bitmaps)
to represent the
* presence or absence of values in a column, enabling efficient querying
and data retrieval.
- * Currently, this type is only applicable to Lance.
+ * Currently, this type is applicable to Lance, is accepted when creating
Doris tables, and is
+ * reported when loading StarRocks tables.
*/
BITMAP,
/**
@@ -103,12 +104,14 @@ public interface Index {
/**
* INVERTED index is a data structure used to optimize full-text searches
by mapping terms to
* their locations within a dataset, allowing for quick retrieval of
documents containing
- * specific words or phrases. Currently, this type is only applicable to
Lance.
+ * specific words or phrases. Currently, this type is applicable to Lance
and Doris, and is
+ * reported when loading StarRocks tables.
*/
INVERTED,
/**
* VECTOR index is used to optimize similarity searches in
high-dimensional vector spaces.
- * Currently, this type is only applicable to Lance.
+ * Currently, this type is applicable to Lance and Doris, and is reported
when loading StarRocks
+ * tables.
*/
VECTOR,
/** IVF_FLAT (Inverted File with Flat quantization) is an indexing method
used for efficient */
diff --git
a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
index 71551d036f..d3a8c2f290 100644
---
a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
+++
b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
@@ -32,10 +32,12 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
@@ -260,12 +262,24 @@ public class StarRocksTableOperations extends
JdbcTableOperations {
try (PreparedStatement preparedStatement =
connection.prepareStatement(sql);
ResultSet resultSet = preparedStatement.executeQuery()) {
+ // SHOW INDEX lists only secondary indexes, one row per single-column
index; table keys are
+ // not included.
List<Index> indexes = new ArrayList<>();
while (resultSet.next()) {
String indexName = resultSet.getString("Key_name");
String columnName = resultSet.getString("Column_name");
- indexes.add(
- Indexes.of(Index.IndexType.PRIMARY_KEY, indexName, new String[][]
{{columnName}}));
+ String starRocksIndexType = resultSet.getString("Index_type");
+ Index.IndexType indexType = mapStarRocksIndexType(starRocksIndexType);
+ if (indexType == null) {
+ LOG.warn(
+ "Skip StarRocks index '{}' on table {}.{} with unsupported type
'{}'",
+ indexName,
+ databaseName,
+ tableName,
+ starRocksIndexType);
+ continue;
+ }
+ indexes.add(Indexes.of(indexType, indexName, new String[][]
{{columnName}}));
}
return indexes;
} catch (SQLException e) {
@@ -630,4 +644,31 @@ public class StarRocksTableOperations extends
JdbcTableOperations {
.build();
return appendColumnDefinition(newColumn, sqlBuilder).toString();
}
+
+ /**
+ * Maps the {@code Index_type} value of StarRocks {@code SHOW INDEX} to a
Gravitino index type.
+ * The value may carry index properties, e.g. {@code GIN("parser" =
"english")}; only the type
+ * name is used and the properties are not kept, since StarRocks indexes
cannot be created or
+ * altered through Gravitino.
+ *
+ * @param indexType the {@code Index_type} value returned by StarRocks
+ * @return the Gravitino index type, or null if the type has no Gravitino
equivalent
+ */
+ @Nullable
+ private static Index.IndexType mapStarRocksIndexType(@Nullable String
indexType) {
+ if (indexType == null) {
+ return null;
+ }
+ String typeName = StringUtils.substringBefore(indexType, "(").trim();
+ switch (typeName.toUpperCase(Locale.ROOT)) {
+ case "BITMAP":
+ return Index.IndexType.BITMAP;
+ case "GIN":
+ return Index.IndexType.INVERTED;
+ case "VECTOR":
+ return Index.IndexType.VECTOR;
+ default:
+ return null;
+ }
+ }
}
diff --git
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
index fb4a4c0ae4..06a5b74200 100644
---
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
+++
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
@@ -846,4 +846,29 @@ public class TestStarRocksTableOperations extends
TestStarRocks {
TABLE_OPERATIONS.drop(databaseName, tableName);
}
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {"PRIMARY", "UNIQUE", "DUPLICATE"})
+ void testLoadTableReportsOnlySecondaryIndexes(String keyType) throws
SQLException {
+ String tableName = GravitinoITUtils.genRandomName("starrocks_index");
+ try (Connection connection = DATA_SOURCE.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(
+ String.format(
+ "CREATE TABLE `%s`.`%s` (`id` INT NOT NULL, `region` VARCHAR(32)
NOT NULL, "
+ + "INDEX `idx_region` (`region`) USING BITMAP) "
+ + "%s KEY(`id`, `region`) DISTRIBUTED BY HASH(`id`) BUCKETS
1 "
+ + "PROPERTIES (\"replication_num\" = \"1\")",
+ databaseName, tableName, keyType));
+ }
+ try {
+ JdbcTable table = TABLE_OPERATIONS.load(databaseName, tableName);
+ Assertions.assertEquals(1, table.index().length);
+ Assertions.assertEquals(Index.IndexType.BITMAP, table.index()[0].type());
+ Assertions.assertEquals("idx_region", table.index()[0].name());
+ Assertions.assertArrayEquals(new String[][] {{"region"}},
table.index()[0].fieldNames());
+ } finally {
+ TABLE_OPERATIONS.drop(databaseName, tableName);
+ }
+ }
}
diff --git
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
index 36ab0c3b84..28dd162b7a 100644
---
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
+++
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
@@ -19,6 +19,9 @@
package org.apache.gravitino.catalog.starrocks.operation;
import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -46,6 +49,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EmptySource;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.NullSource;
+import org.mockito.Mockito;
public class TestStarRocksTableOperationsSqlGeneration {
@@ -100,6 +104,11 @@ public class TestStarRocksTableOperationsSqlGeneration {
return generateAlterTableSql("database", tableName, changes);
}
+ List<Index> indexes(Connection connection, String databaseName, String
tableName)
+ throws SQLException {
+ return getIndexes(connection, databaseName, tableName);
+ }
+
@Override
protected JdbcTable getOrCreateTable(
String databaseName, String tableName, JdbcTable lazyLoadCreateTable) {
@@ -410,6 +419,43 @@ public class TestStarRocksTableOperationsSqlGeneration {
"StarRocks does not support adding an auto-increment column: col2",
exception.getMessage());
}
+ @Test
+ void testGetIndexesMapsTypesAndSkipsUnsupported() throws SQLException {
+ TestableStarRocksTableOperations ops = new
TestableStarRocksTableOperations();
+ Connection connection = Mockito.mock(Connection.class);
+ PreparedStatement statement = Mockito.mock(PreparedStatement.class);
+ ResultSet resultSet = Mockito.mock(ResultSet.class);
+ Mockito.when(connection.prepareStatement("SHOW INDEX FROM `table` FROM
`database`"))
+ .thenReturn(statement);
+ Mockito.when(statement.executeQuery()).thenReturn(resultSet);
+ Mockito.when(resultSet.next()).thenReturn(true, true, true, true, true,
true, false);
+ Mockito.when(resultSet.getString("Key_name"))
+ .thenReturn("idx_region", "idx_ngram", "idx_text", "idx_vector",
"idx_unknown", "idx_null");
+ Mockito.when(resultSet.getString("Column_name"))
+ .thenReturn("region", "name", "text", "embedding", "c5", "c6");
+ Mockito.when(resultSet.getString("Index_type"))
+ .thenReturn(
+ "BITMAP",
+ "NGRAMBF(\"gram_num\" = \"4\")",
+ "GIN(\"parser\" = \"english\")",
+ "VECTOR(\"index_type\" = \"hnsw\", \"dim\" = \"3\")",
+ "UNKNOWN",
+ null);
+
+ List<Index> indexes = ops.indexes(connection, "database", "table");
+
+ Assertions.assertEquals(3, indexes.size());
+ Assertions.assertEquals(Index.IndexType.BITMAP, indexes.get(0).type());
+ Assertions.assertEquals("idx_region", indexes.get(0).name());
+ Assertions.assertArrayEquals(new String[][] {{"region"}},
indexes.get(0).fieldNames());
+ Assertions.assertEquals(Index.IndexType.INVERTED, indexes.get(1).type());
+ Assertions.assertEquals("idx_text", indexes.get(1).name());
+ Assertions.assertArrayEquals(new String[][] {{"text"}},
indexes.get(1).fieldNames());
+ Assertions.assertEquals(Index.IndexType.VECTOR, indexes.get(2).type());
+ Assertions.assertEquals("idx_vector", indexes.get(2).name());
+ Assertions.assertArrayEquals(new String[][] {{"embedding"}},
indexes.get(2).fieldNames());
+ }
+
@Test
void testRenameColumnQuotesIdentifiers() {
TestableStarRocksTableOperations ops = new
TestableStarRocksTableOperations("id", "order-id");
diff --git a/clients/client-python/gravitino/api/rel/indexes/index.py
b/clients/client-python/gravitino/api/rel/indexes/index.py
index 0fe335acc2..2803423490 100644
--- a/clients/client-python/gravitino/api/rel/indexes/index.py
+++ b/clients/client-python/gravitino/api/rel/indexes/index.py
@@ -52,6 +52,15 @@ class Index(ABC):
columns that are not designated as the primary key. The uniqueness
constraint imposed by
UNIQUE KEY helps in avoiding redundancy and ensuring data accuracy in
the database."""
+ BITMAP = "BITMAP"
+ """Bitmap index, which uses bit arrays to represent the presence of
values in a column."""
+
+ INVERTED = "INVERTED"
+ """Inverted index, which maps terms to their locations to speed up
full-text searches."""
+
+ VECTOR = "VECTOR"
+ """Vector index, which speeds up similarity searches in
high-dimensional vector spaces."""
+
DATA_SKIPPING_ANNOY = "DATA_SKIPPING_ANNOY"
"""Legacy ClickHouse Annoy index metadata. This type is
metadata-only."""
diff --git a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
index 99247713ac..263270ecd4 100644
--- a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
+++ b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
@@ -177,6 +177,24 @@ class TestIndexSerdes(unittest.TestCase):
self.assertEqual(Index.IndexType.DATA_SKIPPING_USEARCH,
round_tripped[1].type())
self.assertEqual(usearch_properties, round_tripped[1].properties())
+ def test_index_serdes_bitmap_inverted_vector(self):
+ for index_type in (
+ Index.IndexType.BITMAP,
+ Index.IndexType.INVERTED,
+ Index.IndexType.VECTOR,
+ ):
+ index = IndexSerdes.deserialize(
+ {
+ "indexType": index_type.value,
+ "name": "idx",
+ "fieldNames": [["col"]],
+ }
+ )
+ self.assertEqual(index_type, index.type())
+ self.assertEqual(
+ index_type.value, IndexSerdes.serialize(index)["indexType"]
+ )
+
def test_vector_similarity_index_properties_round_trip(self):
json_string = """
{
diff --git a/docs/jdbc-starrocks-catalog.md b/docs/jdbc-starrocks-catalog.md
index 318ea46114..3bac876332 100644
--- a/docs/jdbc-starrocks-catalog.md
+++ b/docs/jdbc-starrocks-catalog.md
@@ -153,7 +153,15 @@ Unsupported for now.
### Table Indexes
-Unsupported. Creating a table with non-empty index definitions is rejected
before the table is created.
+Creating a table with non-empty index definitions is rejected before the table
is created.
+
+When loading a table, Gravitino reports StarRocks secondary indexes with the
following type mapping. Indexes of other types (e.g. `NGRAMBF`) are not
reported, and table keys are not reported as indexes.
+
+| StarRocks Index Type | Gravitino Index Type |
+|----------------------|----------------------|
+| `BITMAP` | `BITMAP` |
+| `GIN` | `INVERTED` |
+| `VECTOR` | `VECTOR` |
### Table Partitioning
