This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.3 by this push:
new 3631f0d586 [Cherry-pick to branch-1.3] [#14066] fix(starrocks): Map
index types when loading StarRocks tables (#14067) (#14199)
3631f0d586 is described below
commit 3631f0d58673d9dc9d2fabca657d90bb4bebed6f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Oct 26 14:47:14 2026 +0800
[Cherry-pick to branch-1.3] [#14066] fix(starrocks): Map index types when
loading StarRocks tables (#14067) (#14199)
**Cherry-pick Information:**
- Original commit: d4d95ebfe5c92e18e2ae2e8ae88d95aeb0c0251d
- Target branch: `branch-1.3`
- Status: ⚠️ **Has conflicts - manual resolution required**
**Do not merge** until conflict markers are resolved and the
`cherry-pick-conflict` label is removed.
Please review and resolve the conflicts before merging.
Co-authored-by: Yuhui <[email protected]>
Co-authored-by: Jerry Shao <[email protected]>
---
.../org/apache/gravitino/rel/indexes/Index.java | 9 +++--
catalogs/catalog-jdbc-starrocks/build.gradle.kts | 1 +
.../operations/StarRocksTableOperations.java | 45 ++++++++++++++++++++-
.../operation/TestStarRocksTableOperations.java | 25 ++++++++++++
.../TestStarRocksTableOperationsSqlGeneration.java | 46 ++++++++++++++++++++++
.../gravitino/api/rel/indexes/index.py | 9 +++++
.../tests/unittests/dto/rel/test_index_serdes.py | 18 +++++++++
docs/jdbc-starrocks-catalog.md | 10 ++++-
8 files changed, 157 insertions(+), 6 deletions(-)
diff --git a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
index 9f725c19cc..173f87c590 100644
--- a/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
+++ b/api/src/main/java/org/apache/gravitino/rel/indexes/Index.java
@@ -92,7 +92,8 @@ public interface Index {
/**
* Bitmap index is a type of database index that uses bit arrays (bitmaps)
to represent the
* presence or absence of values in a column, enabling efficient querying
and data retrieval.
- * Currently, this type is only applicable to Lance.
+ * Currently, this type is applicable to Lance, is accepted when creating
Doris tables, and is
+ * reported when loading StarRocks tables.
*/
BITMAP,
/**
@@ -103,12 +104,14 @@ public interface Index {
/**
* INVERTED index is a data structure used to optimize full-text searches
by mapping terms to
* their locations within a dataset, allowing for quick retrieval of
documents containing
- * specific words or phrases. Currently, this type is only applicable to
Lance.
+ * specific words or phrases. Currently, this type is applicable to Lance
and Doris, and is
+ * reported when loading StarRocks tables.
*/
INVERTED,
/**
* VECTOR index is used to optimize similarity searches in
high-dimensional vector spaces.
- * Currently, this type is only applicable to Lance.
+ * Currently, this type is applicable to Lance and Doris, and is reported
when loading StarRocks
+ * tables.
*/
VECTOR,
/** IVF_FLAT (Inverted File with Flat quantization) is an indexing method
used for efficient */
diff --git a/catalogs/catalog-jdbc-starrocks/build.gradle.kts
b/catalogs/catalog-jdbc-starrocks/build.gradle.kts
index 03b7d260bb..60a45284da 100644
--- a/catalogs/catalog-jdbc-starrocks/build.gradle.kts
+++ b/catalogs/catalog-jdbc-starrocks/build.gradle.kts
@@ -50,6 +50,7 @@ dependencies {
testImplementation(libs.awaitility)
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.junit.jupiter.params)
+ testImplementation(libs.mockito.core)
testImplementation(libs.mysql.driver)
testImplementation(libs.postgresql.driver)
testImplementation(libs.testcontainers)
diff --git
a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
index 71551d036f..d3a8c2f290 100644
---
a/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
+++
b/catalogs/catalog-jdbc-starrocks/src/main/java/org/apache/gravitino/catalog/starrocks/operations/StarRocksTableOperations.java
@@ -32,10 +32,12 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
@@ -260,12 +262,24 @@ public class StarRocksTableOperations extends
JdbcTableOperations {
try (PreparedStatement preparedStatement =
connection.prepareStatement(sql);
ResultSet resultSet = preparedStatement.executeQuery()) {
+ // SHOW INDEX lists only secondary indexes, one row per single-column
index; table keys are
+ // not included.
List<Index> indexes = new ArrayList<>();
while (resultSet.next()) {
String indexName = resultSet.getString("Key_name");
String columnName = resultSet.getString("Column_name");
- indexes.add(
- Indexes.of(Index.IndexType.PRIMARY_KEY, indexName, new String[][]
{{columnName}}));
+ String starRocksIndexType = resultSet.getString("Index_type");
+ Index.IndexType indexType = mapStarRocksIndexType(starRocksIndexType);
+ if (indexType == null) {
+ LOG.warn(
+ "Skip StarRocks index '{}' on table {}.{} with unsupported type
'{}'",
+ indexName,
+ databaseName,
+ tableName,
+ starRocksIndexType);
+ continue;
+ }
+ indexes.add(Indexes.of(indexType, indexName, new String[][]
{{columnName}}));
}
return indexes;
} catch (SQLException e) {
@@ -630,4 +644,31 @@ public class StarRocksTableOperations extends
JdbcTableOperations {
.build();
return appendColumnDefinition(newColumn, sqlBuilder).toString();
}
+
+ /**
+ * Maps the {@code Index_type} value of StarRocks {@code SHOW INDEX} to a
Gravitino index type.
+ * The value may carry index properties, e.g. {@code GIN("parser" =
"english")}; only the type
+ * name is used and the properties are not kept, since StarRocks indexes
cannot be created or
+ * altered through Gravitino.
+ *
+ * @param indexType the {@code Index_type} value returned by StarRocks
+ * @return the Gravitino index type, or null if the type has no Gravitino
equivalent
+ */
+ @Nullable
+ private static Index.IndexType mapStarRocksIndexType(@Nullable String
indexType) {
+ if (indexType == null) {
+ return null;
+ }
+ String typeName = StringUtils.substringBefore(indexType, "(").trim();
+ switch (typeName.toUpperCase(Locale.ROOT)) {
+ case "BITMAP":
+ return Index.IndexType.BITMAP;
+ case "GIN":
+ return Index.IndexType.INVERTED;
+ case "VECTOR":
+ return Index.IndexType.VECTOR;
+ default:
+ return null;
+ }
+ }
}
diff --git
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
index fb4a4c0ae4..06a5b74200 100644
---
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
+++
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperations.java
@@ -846,4 +846,29 @@ public class TestStarRocksTableOperations extends
TestStarRocks {
TABLE_OPERATIONS.drop(databaseName, tableName);
}
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {"PRIMARY", "UNIQUE", "DUPLICATE"})
+ void testLoadTableReportsOnlySecondaryIndexes(String keyType) throws
SQLException {
+ String tableName = GravitinoITUtils.genRandomName("starrocks_index");
+ try (Connection connection = DATA_SOURCE.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeUpdate(
+ String.format(
+ "CREATE TABLE `%s`.`%s` (`id` INT NOT NULL, `region` VARCHAR(32)
NOT NULL, "
+ + "INDEX `idx_region` (`region`) USING BITMAP) "
+ + "%s KEY(`id`, `region`) DISTRIBUTED BY HASH(`id`) BUCKETS
1 "
+ + "PROPERTIES (\"replication_num\" = \"1\")",
+ databaseName, tableName, keyType));
+ }
+ try {
+ JdbcTable table = TABLE_OPERATIONS.load(databaseName, tableName);
+ Assertions.assertEquals(1, table.index().length);
+ Assertions.assertEquals(Index.IndexType.BITMAP, table.index()[0].type());
+ Assertions.assertEquals("idx_region", table.index()[0].name());
+ Assertions.assertArrayEquals(new String[][] {{"region"}},
table.index()[0].fieldNames());
+ } finally {
+ TABLE_OPERATIONS.drop(databaseName, tableName);
+ }
+ }
}
diff --git
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
index 114f8e50f8..f69fe01191 100644
---
a/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
+++
b/catalogs/catalog-jdbc-starrocks/src/test/java/org/apache/gravitino/catalog/starrocks/operation/TestStarRocksTableOperationsSqlGeneration.java
@@ -19,6 +19,9 @@
package org.apache.gravitino.catalog.starrocks.operation;
import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -46,6 +49,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EmptySource;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.NullSource;
+import org.mockito.Mockito;
public class TestStarRocksTableOperationsSqlGeneration {
@@ -100,6 +104,11 @@ public class TestStarRocksTableOperationsSqlGeneration {
return generateAlterTableSql("database", tableName, changes);
}
+ List<Index> indexes(Connection connection, String databaseName, String
tableName)
+ throws SQLException {
+ return getIndexes(connection, databaseName, tableName);
+ }
+
@Override
protected JdbcTable getOrCreateTable(
String databaseName, String tableName, JdbcTable lazyLoadCreateTable) {
@@ -411,6 +420,43 @@ public class TestStarRocksTableOperationsSqlGeneration {
"StarRocks does not support adding an auto-increment column: col2",
exception.getMessage());
}
+ @Test
+ void testGetIndexesMapsTypesAndSkipsUnsupported() throws SQLException {
+ TestableStarRocksTableOperations ops = new
TestableStarRocksTableOperations();
+ Connection connection = Mockito.mock(Connection.class);
+ PreparedStatement statement = Mockito.mock(PreparedStatement.class);
+ ResultSet resultSet = Mockito.mock(ResultSet.class);
+ Mockito.when(connection.prepareStatement("SHOW INDEX FROM `table` FROM
`database`"))
+ .thenReturn(statement);
+ Mockito.when(statement.executeQuery()).thenReturn(resultSet);
+ Mockito.when(resultSet.next()).thenReturn(true, true, true, true, true,
true, false);
+ Mockito.when(resultSet.getString("Key_name"))
+ .thenReturn("idx_region", "idx_ngram", "idx_text", "idx_vector",
"idx_unknown", "idx_null");
+ Mockito.when(resultSet.getString("Column_name"))
+ .thenReturn("region", "name", "text", "embedding", "c5", "c6");
+ Mockito.when(resultSet.getString("Index_type"))
+ .thenReturn(
+ "BITMAP",
+ "NGRAMBF(\"gram_num\" = \"4\")",
+ "GIN(\"parser\" = \"english\")",
+ "VECTOR(\"index_type\" = \"hnsw\", \"dim\" = \"3\")",
+ "UNKNOWN",
+ null);
+
+ List<Index> indexes = ops.indexes(connection, "database", "table");
+
+ Assertions.assertEquals(3, indexes.size());
+ Assertions.assertEquals(Index.IndexType.BITMAP, indexes.get(0).type());
+ Assertions.assertEquals("idx_region", indexes.get(0).name());
+ Assertions.assertArrayEquals(new String[][] {{"region"}},
indexes.get(0).fieldNames());
+ Assertions.assertEquals(Index.IndexType.INVERTED, indexes.get(1).type());
+ Assertions.assertEquals("idx_text", indexes.get(1).name());
+ Assertions.assertArrayEquals(new String[][] {{"text"}},
indexes.get(1).fieldNames());
+ Assertions.assertEquals(Index.IndexType.VECTOR, indexes.get(2).type());
+ Assertions.assertEquals("idx_vector", indexes.get(2).name());
+ Assertions.assertArrayEquals(new String[][] {{"embedding"}},
indexes.get(2).fieldNames());
+ }
+
@Test
void testRenameColumnQuotesIdentifiers() {
TestableStarRocksTableOperations ops = new
TestableStarRocksTableOperations("id", "order-id");
diff --git a/clients/client-python/gravitino/api/rel/indexes/index.py
b/clients/client-python/gravitino/api/rel/indexes/index.py
index 777e3b4813..e011fc5464 100644
--- a/clients/client-python/gravitino/api/rel/indexes/index.py
+++ b/clients/client-python/gravitino/api/rel/indexes/index.py
@@ -50,6 +50,15 @@ class Index(ABC):
columns that are not designated as the primary key. The uniqueness
constraint imposed by
UNIQUE KEY helps in avoiding redundancy and ensuring data accuracy in
the database."""
+ BITMAP = "BITMAP"
+ """Bitmap index, which uses bit arrays to represent the presence of
values in a column."""
+
+ INVERTED = "INVERTED"
+ """Inverted index, which maps terms to their locations to speed up
full-text searches."""
+
+ VECTOR = "VECTOR"
+ """Vector index, which speeds up similarity searches in
high-dimensional vector spaces."""
+
@abstractmethod
def type(self) -> IndexType:
"""Returns the type of the index. eg: `PRIMARY_KEY` and `UNIQUE_KEY`.
diff --git a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
index 790f102a24..f09e83d141 100644
--- a/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
+++ b/clients/client-python/tests/unittests/dto/rel/test_index_serdes.py
@@ -116,3 +116,21 @@ class TestIndexSerdes(unittest.TestCase):
json_dict = json.loads(json_string)
serialized_dict = json.loads(mock_data_class.to_json())
self.assertDictEqual(json_dict, serialized_dict)
+
+ def test_index_serdes_bitmap_inverted_vector(self):
+ for index_type in (
+ Index.IndexType.BITMAP,
+ Index.IndexType.INVERTED,
+ Index.IndexType.VECTOR,
+ ):
+ index = IndexSerdes.deserialize(
+ {
+ "indexType": index_type.value,
+ "name": "idx",
+ "fieldNames": [["col"]],
+ }
+ )
+ self.assertEqual(index_type, index.type())
+ self.assertEqual(
+ index_type.value, IndexSerdes.serialize(index)["indexType"]
+ )
diff --git a/docs/jdbc-starrocks-catalog.md b/docs/jdbc-starrocks-catalog.md
index ad2d9850ce..6b41d8ac30 100644
--- a/docs/jdbc-starrocks-catalog.md
+++ b/docs/jdbc-starrocks-catalog.md
@@ -149,7 +149,15 @@ Unsupported for now.
### Table Indexes
-Unsupported. Creating a table with non-empty index definitions is rejected
before the table is created.
+Creating a table with non-empty index definitions is rejected before the table
is created.
+
+When loading a table, Gravitino reports StarRocks secondary indexes with the
following type mapping. Indexes of other types (e.g. `NGRAMBF`) are not
reported, and table keys are not reported as indexes.
+
+| StarRocks Index Type | Gravitino Index Type |
+|----------------------|----------------------|
+| `BITMAP` | `BITMAP` |
+| `GIN` | `INVERTED` |
+| `VECTOR` | `VECTOR` |
### Table Partitioning
