This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 4394dfe feat(index): support lumina and vindex global index drop
(#487)
4394dfe is described below
commit 4394dfe79618146a892afed945367a80b2032344
Author: Junrui Lee <[email protected]>
AuthorDate: Thu Jul 9 10:31:09 2026 +0800
feat(index): support lumina and vindex global index drop (#487)
---
crates/integrations/datafusion/src/procedures.rs | 18 +--
crates/integrations/datafusion/tests/procedures.rs | 36 +++++-
...rop_builder.rs => global_index_drop_builder.rs} | 143 ++++++++++++++++++---
crates/paimon/src/table/global_index_types.rs | 104 +++++++++++++++
crates/paimon/src/table/mod.rs | 11 +-
5 files changed, 284 insertions(+), 28 deletions(-)
diff --git a/crates/integrations/datafusion/src/procedures.rs
b/crates/integrations/datafusion/src/procedures.rs
index 76f3660..32b7989 100644
--- a/crates/integrations/datafusion/src/procedures.rs
+++ b/crates/integrations/datafusion/src/procedures.rs
@@ -26,8 +26,7 @@
//! - `CALL sys.create_global_index(table => '...', index_column => '...',
index_type => 'btree')`
//! - `CALL sys.create_global_index(table => '...', index_column => '...',
index_type => 'bitmap')`
//! - `CALL sys.create_global_index(table => '...', index_column => '...',
index_type => 'ivf-pq')`
-//! - `CALL sys.drop_global_index(table => '...', index_column => '...',
index_type => 'btree')`
-//! - `CALL sys.drop_global_index(table => '...', index_column => '...',
index_type => 'bitmap')`
+//! - `CALL sys.drop_global_index(table => '...', index_column => '...',
index_type => 'btree')` (also 'bitmap', 'lumina', or a vindex type such as
'ivf-pq')
//! - `CALL sys.create_lumina_index(table => '...', index_column => '...')`
use std::collections::HashMap;
@@ -44,7 +43,10 @@ use datafusion::sql::sqlparser::ast::{
};
use paimon::catalog::{Catalog, Identifier};
use paimon::spec::Snapshot;
-use paimon::table::{SnapshotManager, Table, TagManager};
+use paimon::table::{
+ normalize_global_index_type_for_drop, SnapshotManager, Table, TagManager,
+ SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP,
+};
use paimon::vindex::is_vindex_index_type;
use crate::error::to_datafusion_error;
@@ -587,23 +589,23 @@ async fn proc_drop_global_index(
.get("index_type")
.map(String::as_str)
.unwrap_or("btree");
- if !is_sorted_global_index_type(index_type) {
+ if normalize_global_index_type_for_drop(index_type).is_none() {
return Err(DataFusionError::NotImplemented(format!(
- "drop_global_index only supports index_type => 'btree' or
'bitmap', got '{index_type}'"
+ "unsupported global index type '{index_type}'; supported:
{SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP}"
)));
}
if args.contains_key("partitions") {
return Err(DataFusionError::NotImplemented(
- "drop_global_index partitions are not supported for btree
yet".to_string(),
+ "drop_global_index partitions are not supported yet".to_string(),
));
}
if args.contains_key("dry_run") {
return Err(DataFusionError::NotImplemented(
- "drop_global_index dry_run is not supported for btree
yet".to_string(),
+ "drop_global_index dry_run is not supported yet".to_string(),
));
}
- let mut builder = table.new_btree_global_index_drop_builder();
+ let mut builder = table.new_global_index_drop_builder();
builder.with_index_column(index_column);
builder.with_index_type(index_type);
builder.execute().await.map_err(to_datafusion_error)?;
diff --git a/crates/integrations/datafusion/tests/procedures.rs
b/crates/integrations/datafusion/tests/procedures.rs
index 15ac8d1..90d4d44 100644
--- a/crates/integrations/datafusion/tests/procedures.rs
+++ b/crates/integrations/datafusion/tests/procedures.rs
@@ -373,7 +373,41 @@ async fn
test_drop_global_index_rejects_unsupported_index_type() {
assert_sql_error(
&sql_context,
"CALL sys.drop_global_index(table =>
'test_db.global_index_drop_bad_type', index_column => 'id', index_type =>
'full-text')",
- "only supports index_type => 'btree' or 'bitmap'",
+ "unsupported global index type",
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn test_drop_global_index_accepts_lumina_type() {
+ let (_tmp, sql_context) =
setup_btree_global_index_table("lumina_drop_accepted").await;
+ exec(
+ &sql_context,
+ "INSERT INTO paimon.test_db.lumina_drop_accepted (id, name) VALUES (1,
'alice')",
+ )
+ .await;
+
+ // No lumina index exists; the call must be accepted and succeed (no-match
Ok),
+ // NOT rejected as an unsupported type.
+ exec(
+ &sql_context,
+ "CALL sys.drop_global_index(table => 'test_db.lumina_drop_accepted',
index_column => 'id', index_type => 'lumina')",
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn test_drop_global_index_accepts_vindex_type() {
+ let (_tmp, sql_context) =
setup_btree_global_index_table("vindex_drop_accepted").await;
+ exec(
+ &sql_context,
+ "INSERT INTO paimon.test_db.vindex_drop_accepted (id, name) VALUES (1,
'alice')",
+ )
+ .await;
+
+ exec(
+ &sql_context,
+ "CALL sys.drop_global_index(table => 'test_db.vindex_drop_accepted',
index_column => 'id', index_type => 'ivf-pq')",
)
.await;
}
diff --git a/crates/paimon/src/table/btree_global_index_drop_builder.rs
b/crates/paimon/src/table/global_index_drop_builder.rs
similarity index 71%
rename from crates/paimon/src/table/btree_global_index_drop_builder.rs
rename to crates/paimon/src/table/global_index_drop_builder.rs
index a1c497e..9d6f557 100644
--- a/crates/paimon/src/table/btree_global_index_drop_builder.rs
+++ b/crates/paimon/src/table/global_index_drop_builder.rs
@@ -15,19 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-use super::global_index_types::{normalize_sorted_global_index_type,
BTREE_GLOBAL_INDEX_TYPE};
+use super::global_index_types::{
+ normalize_global_index_type_for_drop, BTREE_GLOBAL_INDEX_TYPE,
+ SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP,
+};
use crate::spec::{DataField, FileKind, IndexFileMeta, IndexManifest};
use crate::table::{CommitMessage, SnapshotManager, Table, TableCommit};
use crate::{Error, Result};
use std::collections::HashMap;
-pub struct BTreeGlobalIndexDropBuilder<'a> {
+pub struct GlobalIndexDropBuilder<'a> {
table: &'a Table,
index_column: Option<String>,
index_type: String,
}
-impl<'a> BTreeGlobalIndexDropBuilder<'a> {
+impl<'a> GlobalIndexDropBuilder<'a> {
pub(crate) fn new(table: &'a Table) -> Self {
Self {
table,
@@ -47,19 +50,20 @@ impl<'a> BTreeGlobalIndexDropBuilder<'a> {
}
pub async fn execute(&self) -> Result<usize> {
- let index_type =
normalize_sorted_global_index_type(&self.index_type).ok_or_else(|| {
- Error::Unsupported {
- message: format!(
- "Sorted global index drop only supports index_type =>
'btree' or 'bitmap', got '{}'",
- self.index_type
- ),
- }
- })?;
+ let index_type =
+
normalize_global_index_type_for_drop(&self.index_type).ok_or_else(|| {
+ Error::Unsupported {
+ message: format!(
+ "unsupported global index type '{}'; supported: {}",
+ self.index_type, SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP
+ ),
+ }
+ })?;
let index_column = self
.index_column
.as_deref()
.ok_or_else(|| Error::DataInvalid {
- message: "Sorted global index column is required".to_string(),
+ message: "Global index column is required".to_string(),
source: None,
})?;
let index_field = find_index_field(self.table, index_column)?;
@@ -84,7 +88,12 @@ impl<'a> BTreeGlobalIndexDropBuilder<'a> {
HashMap::new();
let mut dropped = 0;
for entry in index_entries {
- if entry.kind != FileKind::Add || entry.index_file.index_type !=
index_type {
+ if entry.kind != FileKind::Add {
+ continue;
+ }
+ if
normalize_global_index_type_for_drop(&entry.index_file.index_type)
+ != Some(index_type)
+ {
continue;
}
let Some(global_meta) =
entry.index_file.global_index_meta.as_ref() else {
@@ -310,7 +319,7 @@ mod tests {
.unwrap();
let dropped = table
- .new_btree_global_index_drop_builder()
+ .new_global_index_drop_builder()
.with_index_column("id")
.execute()
.await
@@ -353,7 +362,7 @@ mod tests {
.unwrap();
let dropped = table
- .new_btree_global_index_drop_builder()
+ .new_global_index_drop_builder()
.with_index_column("id")
.execute()
.await
@@ -364,4 +373,108 @@ mod tests {
assert_eq!(remaining.len(), 1);
assert_eq!(remaining[0].index_file.file_name, "btree-name.index");
}
+
+ #[tokio::test]
+ async fn test_drop_lumina_matches_legacy_alias_entry() {
+ let table = test_table("memory:/test_drop_lumina_legacy");
+ setup_dirs(&table).await;
+
+ let mut message = CommitMessage::new(
+ BinaryRow::new(0).to_serialized_bytes(),
+ 0,
+ vec![data_file("data-0.parquet")],
+ );
+ // Stored under the LEGACY identifier; request uses the canonical name.
+ message.new_index_files = vec![
+ global_index_file("lumina-vector-ann", "lumina-id.index", 0, 0, 9),
+ global_index_file(BTREE_GLOBAL_INDEX_TYPE, "btree-id.index", 0, 0,
9),
+ ];
+ TableCommit::new(table.clone(), "test-user".to_string())
+ .commit(vec![message])
+ .await
+ .unwrap();
+
+ let dropped = table
+ .new_global_index_drop_builder()
+ .with_index_column("id")
+ .with_index_type("lumina")
+ .execute()
+ .await
+ .unwrap();
+ assert_eq!(dropped, 1);
+
+ let mut remaining = latest_index_entries(&table)
+ .await
+ .into_iter()
+ .map(|entry| entry.index_file.file_name)
+ .collect::<Vec<_>>();
+ remaining.sort();
+ assert_eq!(remaining, vec!["btree-id.index".to_string()]);
+ }
+
+ #[tokio::test]
+ async fn test_drop_vindex_preserves_other_types_case_insensitive() {
+ let table = test_table("memory:/test_drop_vindex_identity");
+ setup_dirs(&table).await;
+
+ let mut message = CommitMessage::new(
+ BinaryRow::new(0).to_serialized_bytes(),
+ 0,
+ vec![data_file("data-0.parquet")],
+ );
+ message.new_index_files = vec![
+ global_index_file("ivf-flat", "ivf-flat-id.index", 0, 0, 9),
+ global_index_file("ivf-pq", "ivf-pq-id.index", 0, 0, 9),
+ global_index_file("lumina", "lumina-id.index", 0, 0, 9),
+ global_index_file(BTREE_GLOBAL_INDEX_TYPE, "btree-id.index", 0, 0,
9),
+ ];
+ TableCommit::new(table.clone(), "test-user".to_string())
+ .commit(vec![message])
+ .await
+ .unwrap();
+
+ // Uppercase request must canonicalize and drop ONLY ivf-flat.
+ let dropped = table
+ .new_global_index_drop_builder()
+ .with_index_column("id")
+ .with_index_type("IVF-FLAT")
+ .execute()
+ .await
+ .unwrap();
+ assert_eq!(dropped, 1);
+
+ let mut remaining = latest_index_entries(&table)
+ .await
+ .into_iter()
+ .map(|entry| entry.index_file.file_name)
+ .collect::<Vec<_>>();
+ remaining.sort();
+ assert_eq!(
+ remaining,
+ vec![
+ "btree-id.index".to_string(),
+ "ivf-pq-id.index".to_string(),
+ "lumina-id.index".to_string(),
+ ]
+ );
+ }
+
+ #[tokio::test]
+ async fn test_drop_unsupported_type_errors() {
+ let table = test_table("memory:/test_drop_unsupported");
+ setup_dirs(&table).await;
+
+ let err = table
+ .new_global_index_drop_builder()
+ .with_index_column("id")
+ .with_index_type("full-text")
+ .execute()
+ .await
+ .expect_err("unsupported type must error");
+ assert!(matches!(
+ err,
+ Error::Unsupported { message }
+ if message.contains("unsupported global index type") &&
message.contains("ivf-pq")
+ ));
+ }
}
diff --git a/crates/paimon/src/table/global_index_types.rs
b/crates/paimon/src/table/global_index_types.rs
index bfa4430..ffd91a8 100644
--- a/crates/paimon/src/table/global_index_types.rs
+++ b/crates/paimon/src/table/global_index_types.rs
@@ -15,6 +15,12 @@
// specific language governing permissions and limitations
// under the License.
+use crate::lumina::{is_lumina_index_type, LUMINA_IDENTIFIER};
+use crate::vindex::{
+ is_vindex_index_type, IVF_FLAT_IDENTIFIER, IVF_HNSW_FLAT_IDENTIFIER,
IVF_HNSW_SQ_IDENTIFIER,
+ IVF_PQ_IDENTIFIER,
+};
+
pub(crate) const BTREE_GLOBAL_INDEX_TYPE: &str = "btree";
pub(crate) const BITMAP_GLOBAL_INDEX_TYPE: &str = "bitmap";
@@ -27,3 +33,101 @@ pub(crate) fn
normalize_sorted_global_index_type(index_type: &str) -> Option<&'s
None
}
}
+
+/// Human-readable list of the global index types the Rust drop path accepts.
+/// Used verbatim in the unsupported-type error of both the builder and the
+/// DataFusion procedure so the two messages stay in sync.
+pub const SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP: &str =
+ "btree, bitmap, lumina, lumina-vector-ann, ivf-flat, ivf-pq,
ivf-hnsw-flat, ivf-hnsw-sq";
+
+/// Canonicalize any supported global index type to a stable `&'static str`, or
+/// `None` if unsupported. Case-insensitive. Order: sorted -> lumina -> vindex.
+///
+/// Both lumina aliases (`lumina`, `lumina-vector-ann`) canonicalize to
+/// `"lumina"`; each vindex type keeps its own identity so dropping one vindex
+/// type never matches another on the same column. Callers should compare the
+/// canonical form of BOTH the request and each stored entry's `index_type`.
+pub fn normalize_global_index_type_for_drop(index_type: &str) ->
Option<&'static str> {
+ if let Some(sorted) = normalize_sorted_global_index_type(index_type) {
+ return Some(sorted);
+ }
+ // is_lumina_index_type / is_vindex_index_type match case-sensitively, so
+ // lowercase first (normalize_sorted_global_index_type is already
case-insensitive).
+ let lowered = index_type.to_ascii_lowercase();
+ if is_lumina_index_type(&lowered) {
+ return Some(LUMINA_IDENTIFIER);
+ }
+ if is_vindex_index_type(&lowered) {
+ return canonical_vindex_identifier(&lowered);
+ }
+ None
+}
+
+// Map a lowercased vindex identifier to its &'static constant (never return
the
+// borrowed `lowered` — wrong lifetime and not canonical).
+fn canonical_vindex_identifier(lowered: &str) -> Option<&'static str> {
+ match lowered {
+ IVF_FLAT_IDENTIFIER => Some(IVF_FLAT_IDENTIFIER),
+ IVF_PQ_IDENTIFIER => Some(IVF_PQ_IDENTIFIER),
+ IVF_HNSW_FLAT_IDENTIFIER => Some(IVF_HNSW_FLAT_IDENTIFIER),
+ IVF_HNSW_SQ_IDENTIFIER => Some(IVF_HNSW_SQ_IDENTIFIER),
+ _ => None,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn sorted_types_canonicalize_case_insensitively() {
+ assert_eq!(normalize_global_index_type_for_drop("BTREE"),
Some("btree"));
+ assert_eq!(
+ normalize_global_index_type_for_drop("Bitmap"),
+ Some("bitmap")
+ );
+ }
+
+ #[test]
+ fn lumina_aliases_canonicalize_to_lumina() {
+ assert_eq!(
+ normalize_global_index_type_for_drop("lumina"),
+ Some("lumina")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("lumina-vector-ann"),
+ Some("lumina")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("LUMINA"),
+ Some("lumina")
+ );
+ }
+
+ #[test]
+ fn vindex_types_keep_distinct_identity() {
+ assert_eq!(
+ normalize_global_index_type_for_drop("ivf-flat"),
+ Some("ivf-flat")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("IVF-PQ"),
+ Some("ivf-pq")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("ivf-hnsw-flat"),
+ Some("ivf-hnsw-flat")
+ );
+ assert_eq!(
+ normalize_global_index_type_for_drop("ivf-hnsw-sq"),
+ Some("ivf-hnsw-sq")
+ );
+ }
+
+ #[test]
+ fn unsupported_types_return_none() {
+ assert_eq!(normalize_global_index_type_for_drop("full-text"), None);
+ assert_eq!(normalize_global_index_type_for_drop("hash"), None);
+ assert_eq!(normalize_global_index_type_for_drop(""), None);
+ }
+}
diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs
index 1865593..e28e6f4 100644
--- a/crates/paimon/src/table/mod.rs
+++ b/crates/paimon/src/table/mod.rs
@@ -23,7 +23,6 @@ mod bitmap_global_index_reader;
mod blob_resolver;
mod branch_manager;
mod btree_global_index_build_builder;
-mod btree_global_index_drop_builder;
mod bucket_assigner;
mod bucket_assigner_constant;
mod bucket_assigner_cross;
@@ -41,6 +40,7 @@ mod dedicated_format_file_writer;
#[cfg(feature = "fulltext")]
mod full_text_search_builder;
pub(crate) mod global_index_build_common;
+mod global_index_drop_builder;
pub(crate) mod global_index_scanner;
mod global_index_types;
mod hybrid_search_builder;
@@ -77,13 +77,16 @@ use crate::Result;
use arrow_array::RecordBatch;
pub use branch_manager::BranchManager;
pub use btree_global_index_build_builder::BTreeGlobalIndexBuildBuilder;
-pub use btree_global_index_drop_builder::BTreeGlobalIndexDropBuilder;
pub use commit_message::CommitMessage;
pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo};
pub use data_evolution_writer::{DataEvolutionDeleteWriter,
DataEvolutionWriter};
#[cfg(feature = "fulltext")]
pub use full_text_search_builder::FullTextSearchBuilder;
use futures::stream::BoxStream;
+pub use global_index_drop_builder::GlobalIndexDropBuilder;
+pub use global_index_types::{
+ normalize_global_index_type_for_drop,
SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP,
+};
pub use hybrid_search_builder::{
HybridSearchBuilder, HybridSearchRanker, HybridSearchRoute,
HybridSearchRouteKind,
};
@@ -220,8 +223,8 @@ impl Table {
BTreeGlobalIndexBuildBuilder::new(self)
}
- pub fn new_btree_global_index_drop_builder(&self) ->
BTreeGlobalIndexDropBuilder<'_> {
- BTreeGlobalIndexDropBuilder::new(self)
+ pub fn new_global_index_drop_builder(&self) -> GlobalIndexDropBuilder<'_> {
+ GlobalIndexDropBuilder::new(self)
}
pub fn new_vindex_index_build_builder(&self, index_type: &str) ->
VindexIndexBuildBuilder<'_> {
