This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 34449a72 chore: upgrade paimon-vindex-core to 0.3.0 (#630)
34449a72 is described below
commit 34449a72d448814a69b065ded1df0fb98b933989
Author: Jingsong Lee <[email protected]>
AuthorDate: Thu Jul 30 13:10:04 2026 +0800
chore: upgrade paimon-vindex-core to 0.3.0 (#630)
---
Cargo.lock | 5 +--
DEPENDENCIES.rust.tsv | 2 +-
benchmarks/tpcds/DEPENDENCIES.rust.tsv | 2 +-
bindings/c/Cargo.toml | 2 +-
bindings/c/DEPENDENCIES.rust.tsv | 2 +-
bindings/go/DEPENDENCIES.rust.tsv | 2 +-
bindings/python/DEPENDENCIES.rust.tsv | 2 +-
crates/integration_tests/DEPENDENCIES.rust.tsv | 2 +-
.../integrations/datafusion/DEPENDENCIES.rust.tsv | 2 +-
crates/integrations/datafusion/src/procedures.rs | 2 +-
crates/integrations/datafusion/tests/procedures.rs | 22 ++++++++----
crates/paimon-rest-server/DEPENDENCIES.rust.tsv | 2 +-
crates/paimon/Cargo.toml | 2 +-
crates/paimon/DEPENDENCIES.rust.tsv | 2 +-
crates/paimon/src/table/global_index_types.rs | 19 +++-------
crates/paimon/src/vindex/mod.rs | 42 +++-------------------
crates/paimon/src/vindex/reader.rs | 24 +++----------
docs/src/sql.md | 9 ++---
18 files changed, 47 insertions(+), 98 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 47eb4598..69d011fa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4704,10 +4704,11 @@ dependencies = [
[[package]]
name = "paimon-vindex-core"
-version = "0.2.0"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ed02b73fbd5cfbe20df6b66bca52f5ba66e3c29bd00ad450fbc02f9e94242a4b"
+checksum = "b2c67c916596e578ed09f78ace8b4c54fc45d262933c658e1ca7fdb61b722635"
dependencies = [
+ "half",
"matrixmultiply",
"nalgebra",
"rand 0.8.7",
diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv
index 7e1cfad1..7b2d7772 100644
--- a/DEPENDENCIES.rust.tsv
+++ b/DEPENDENCIES.rust.tsv
@@ -394,7 +394,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv
b/benchmarks/tpcds/DEPENDENCIES.rust.tsv
index 8efd4d78..da26dc28 100644
--- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv
+++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv
@@ -266,7 +266,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml
index 2340f970..af0a5121 100644
--- a/bindings/c/Cargo.toml
+++ b/bindings/c/Cargo.toml
@@ -43,4 +43,4 @@ serde_json = "1.0.120"
# Test-only: the vector-search integration tests build a real primary-key
vindex
# IVF-flat ANN segment fixture in-process. Versions match crates/paimon.
bytes = "1.7.1"
-paimon-vindex-core = "0.2.0"
+paimon-vindex-core = "0.3.0"
diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv
index b2da9a11..f196c1cf 100644
--- a/bindings/c/DEPENDENCIES.rust.tsv
+++ b/bindings/c/DEPENDENCIES.rust.tsv
@@ -198,7 +198,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/bindings/go/DEPENDENCIES.rust.tsv
b/bindings/go/DEPENDENCIES.rust.tsv
index b2da9a11..f196c1cf 100644
--- a/bindings/go/DEPENDENCIES.rust.tsv
+++ b/bindings/go/DEPENDENCIES.rust.tsv
@@ -198,7 +198,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/bindings/python/DEPENDENCIES.rust.tsv
b/bindings/python/DEPENDENCIES.rust.tsv
index 2514af89..f4d35ed9 100644
--- a/bindings/python/DEPENDENCIES.rust.tsv
+++ b/bindings/python/DEPENDENCIES.rust.tsv
@@ -323,7 +323,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv
b/crates/integration_tests/DEPENDENCIES.rust.tsv
index 8ba30423..fffee985 100644
--- a/crates/integration_tests/DEPENDENCIES.rust.tsv
+++ b/crates/integration_tests/DEPENDENCIES.rust.tsv
@@ -198,7 +198,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
index f337294b..8b242ee9 100644
--- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
+++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv
@@ -342,7 +342,7 @@ [email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/crates/integrations/datafusion/src/procedures.rs
b/crates/integrations/datafusion/src/procedures.rs
index 32b7989a..9d2d9196 100644
--- a/crates/integrations/datafusion/src/procedures.rs
+++ b/crates/integrations/datafusion/src/procedures.rs
@@ -571,7 +571,7 @@ async fn proc_create_global_index(
} else {
return Err(DataFusionError::NotImplemented(format!(
"create_global_index only supports index_type => 'btree',
'bitmap', or vindex types \
- ('ivf-flat', 'ivf-pq', 'ivf-hnsw-flat', 'ivf-hnsw-sq'), got
'{index_type}'"
+ ('ivf-flat', 'ivf-pq'), got '{index_type}'"
)));
}
ok_result(ctx)
diff --git a/crates/integrations/datafusion/tests/procedures.rs
b/crates/integrations/datafusion/tests/procedures.rs
index 90d4d44e..be2ddcbe 100644
--- a/crates/integrations/datafusion/tests/procedures.rs
+++ b/crates/integrations/datafusion/tests/procedures.rs
@@ -153,15 +153,23 @@ async fn test_create_global_index_requires_index_column()
{
}
#[tokio::test]
-async fn test_create_global_index_rejects_unsupported_index_type() {
+async fn test_create_global_index_rejects_unsupported_index_types() {
let (_tmp, sql_context) =
setup_btree_global_index_table("global_index_bad_type").await;
- assert_sql_error(
- &sql_context,
- "CALL sys.create_global_index(table =>
'test_db.global_index_bad_type', index_column => 'id', index_type =>
'full-text')",
- "only supports index_type => 'btree', 'bitmap'",
- )
- .await;
+ for index_type in ["full-text", "ivf-hnsw-flat", "ivf-hnsw-sq"] {
+ assert_sql_error(
+ &sql_context,
+ &format!(
+ "CALL sys.create_global_index(\
+ table => 'test_db.global_index_bad_type', \
+ index_column => 'id', \
+ index_type => '{index_type}'\
+ )"
+ ),
+ "only supports index_type => 'btree', 'bitmap', or vindex types
('ivf-flat', 'ivf-pq')",
+ )
+ .await;
+ }
}
#[tokio::test]
diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
index 75e5051a..fbfc944b 100644
--- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
+++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv
@@ -202,7 +202,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml
index b84e7dcd..1bed943a 100644
--- a/crates/paimon/Cargo.toml
+++ b/crates/paimon/Cargo.toml
@@ -115,7 +115,7 @@ urlencoding = "2.1"
paimon-mosaic-core = "0.2.0"
paimon-ftindex-core = { version = "0.1.0", optional = true }
tempfile = { version = "3", optional = true }
-paimon-vindex-core = "0.2.0"
+paimon-vindex-core = "0.3.0"
vortex = { version = "0.75.0", features = ["tokio"], optional = true }
libloading = "0.9"
log = "0.4"
diff --git a/crates/paimon/DEPENDENCIES.rust.tsv
b/crates/paimon/DEPENDENCIES.rust.tsv
index 2c28cf77..1cce09cf 100644
--- a/crates/paimon/DEPENDENCIES.rust.tsv
+++ b/crates/paimon/DEPENDENCIES.rust.tsv
@@ -318,7 +318,7 @@ [email protected]
X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
[email protected] X
X
[email protected] X
X
[email protected] X
X
diff --git a/crates/paimon/src/table/global_index_types.rs
b/crates/paimon/src/table/global_index_types.rs
index ffd91a81..eb83ad72 100644
--- a/crates/paimon/src/table/global_index_types.rs
+++ b/crates/paimon/src/table/global_index_types.rs
@@ -16,10 +16,7 @@
// under the License.
use crate::lumina::{is_lumina_index_type, LUMINA_IDENTIFIER};
-use crate::vindex::{
- is_vindex_index_type, IVF_FLAT_IDENTIFIER, IVF_HNSW_FLAT_IDENTIFIER,
IVF_HNSW_SQ_IDENTIFIER,
- IVF_PQ_IDENTIFIER,
-};
+use crate::vindex::{is_vindex_index_type, IVF_FLAT_IDENTIFIER,
IVF_PQ_IDENTIFIER};
pub(crate) const BTREE_GLOBAL_INDEX_TYPE: &str = "btree";
pub(crate) const BITMAP_GLOBAL_INDEX_TYPE: &str = "bitmap";
@@ -38,7 +35,7 @@ pub(crate) fn normalize_sorted_global_index_type(index_type:
&str) -> Option<&'s
/// Used verbatim in the unsupported-type error of both the builder and the
/// DataFusion procedure so the two messages stay in sync.
pub const SUPPORTED_GLOBAL_INDEX_TYPES_FOR_DROP: &str =
- "btree, bitmap, lumina, lumina-vector-ann, ivf-flat, ivf-pq,
ivf-hnsw-flat, ivf-hnsw-sq";
+ "btree, bitmap, lumina, lumina-vector-ann, ivf-flat, ivf-pq";
/// Canonicalize any supported global index type to a stable `&'static str`, or
/// `None` if unsupported. Case-insensitive. Order: sorted -> lumina -> vindex.
@@ -69,8 +66,6 @@ fn canonical_vindex_identifier(lowered: &str) ->
Option<&'static str> {
match lowered {
IVF_FLAT_IDENTIFIER => Some(IVF_FLAT_IDENTIFIER),
IVF_PQ_IDENTIFIER => Some(IVF_PQ_IDENTIFIER),
- IVF_HNSW_FLAT_IDENTIFIER => Some(IVF_HNSW_FLAT_IDENTIFIER),
- IVF_HNSW_SQ_IDENTIFIER => Some(IVF_HNSW_SQ_IDENTIFIER),
_ => None,
}
}
@@ -114,20 +109,14 @@ mod tests {
normalize_global_index_type_for_drop("IVF-PQ"),
Some("ivf-pq")
);
- assert_eq!(
- normalize_global_index_type_for_drop("ivf-hnsw-flat"),
- Some("ivf-hnsw-flat")
- );
- assert_eq!(
- normalize_global_index_type_for_drop("ivf-hnsw-sq"),
- Some("ivf-hnsw-sq")
- );
}
#[test]
fn unsupported_types_return_none() {
assert_eq!(normalize_global_index_type_for_drop("full-text"), None);
assert_eq!(normalize_global_index_type_for_drop("hash"), None);
+ assert_eq!(normalize_global_index_type_for_drop("ivf-hnsw-flat"),
None);
+ assert_eq!(normalize_global_index_type_for_drop("ivf-hnsw-sq"), None);
assert_eq!(normalize_global_index_type_for_drop(""), None);
}
}
diff --git a/crates/paimon/src/vindex/mod.rs b/crates/paimon/src/vindex/mod.rs
index 77457efd..f6e35ab5 100644
--- a/crates/paimon/src/vindex/mod.rs
+++ b/crates/paimon/src/vindex/mod.rs
@@ -25,8 +25,6 @@ use std::collections::HashMap;
pub const IVF_FLAT_IDENTIFIER: &str = "ivf-flat";
pub const IVF_PQ_IDENTIFIER: &str = "ivf-pq";
-pub const IVF_HNSW_FLAT_IDENTIFIER: &str = "ivf-hnsw-flat";
-pub const IVF_HNSW_SQ_IDENTIFIER: &str = "ivf-hnsw-sq";
const DEFAULT_DIMENSION: &str = "128";
const DEFAULT_METRIC: &str = "inner_product";
@@ -35,18 +33,13 @@ const DEFAULT_PQ_M: &str = "16";
const DEFAULT_PQ_USE_OPQ: &str = "false";
pub fn is_vindex_index_type(index_type: &str) -> bool {
- matches!(
- index_type,
- IVF_FLAT_IDENTIFIER | IVF_PQ_IDENTIFIER | IVF_HNSW_FLAT_IDENTIFIER |
IVF_HNSW_SQ_IDENTIFIER
- )
+ matches!(index_type, IVF_FLAT_IDENTIFIER | IVF_PQ_IDENTIFIER)
}
pub(crate) fn native_index_type(index_type: &str) -> Option<&'static str> {
match index_type {
IVF_FLAT_IDENTIFIER => Some("ivf_flat"),
IVF_PQ_IDENTIFIER => Some("ivf_pq"),
- IVF_HNSW_FLAT_IDENTIFIER => Some("ivf_hnsw_flat"),
- IVF_HNSW_SQ_IDENTIFIER => Some("ivf_hnsw_sq"),
_ => None,
}
}
@@ -131,19 +124,6 @@ impl VindexVectorIndexOptions {
);
}
- for key in ["hnsw.m", "hnsw.ef-construction", "hnsw.max-level"] {
- if let Some(value) = optional_value(
- table_options,
- user_options,
- field.name(),
- index_type,
- key,
- key,
- ) {
- native_options.insert(key.to_string(), value);
- }
- }
-
let config =
VectorIndexConfig::from_options(&native_options).map_err(|e| {
crate::Error::DataInvalid {
message: format!("Invalid vindex options: {e}"),
@@ -232,9 +212,6 @@ fn is_allowed_native_key(key: &str, index_type: &str) ->
bool {
match key {
"dimension" | "nlist" | "metric" => true,
"pq.m" | "use-opq" => index_type == IVF_PQ_IDENTIFIER,
- "hnsw.m" | "hnsw.ef-construction" | "hnsw.max-level" => {
- index_type == IVF_HNSW_FLAT_IDENTIFIER || index_type ==
IVF_HNSW_SQ_IDENTIFIER
- }
_ => false,
}
}
@@ -243,9 +220,6 @@ fn is_allowed_paimon_suffix(suffix: &str, index_type: &str)
-> bool {
match suffix {
"dimension" | "nlist" | "distance.metric" => true,
"pq.m" | "pq.use-opq" => index_type == IVF_PQ_IDENTIFIER,
- "hnsw.m" | "hnsw.ef-construction" | "hnsw.max-level" => {
- index_type == IVF_HNSW_FLAT_IDENTIFIER || index_type ==
IVF_HNSW_SQ_IDENTIFIER
- }
_ => false,
}
}
@@ -334,8 +308,8 @@ mod tests {
fn test_vindex_index_type_identifier_helper() {
assert!(is_vindex_index_type(IVF_FLAT_IDENTIFIER));
assert!(is_vindex_index_type(IVF_PQ_IDENTIFIER));
- assert!(is_vindex_index_type(IVF_HNSW_FLAT_IDENTIFIER));
- assert!(is_vindex_index_type(IVF_HNSW_SQ_IDENTIFIER));
+ assert!(!is_vindex_index_type("ivf-hnsw-flat"));
+ assert!(!is_vindex_index_type("ivf-hnsw-sq"));
assert!(!is_vindex_index_type(""));
assert!(!is_vindex_index_type("btree"));
assert!(!is_vindex_index_type("lumina"));
@@ -546,14 +520,8 @@ mod tests {
fn test_native_index_type_helper() {
assert_eq!(native_index_type(IVF_FLAT_IDENTIFIER), Some("ivf_flat"));
assert_eq!(native_index_type(IVF_PQ_IDENTIFIER), Some("ivf_pq"));
- assert_eq!(
- native_index_type(IVF_HNSW_FLAT_IDENTIFIER),
- Some("ivf_hnsw_flat")
- );
- assert_eq!(
- native_index_type(IVF_HNSW_SQ_IDENTIFIER),
- Some("ivf_hnsw_sq")
- );
+ assert_eq!(native_index_type("ivf-hnsw-flat"), None);
+ assert_eq!(native_index_type("ivf-hnsw-sq"), None);
assert_eq!(native_index_type("btree"), None);
}
diff --git a/crates/paimon/src/vindex/reader.rs
b/crates/paimon/src/vindex/reader.rs
index c4d6abe7..74551154 100644
--- a/crates/paimon/src/vindex/reader.rs
+++ b/crates/paimon/src/vindex/reader.rs
@@ -25,9 +25,7 @@ use std::collections::HashMap;
use std::io::{Cursor, Read, Seek, SeekFrom};
const DEFAULT_NPROBE: usize = 16;
-const DEFAULT_EF_SEARCH: usize = 0;
const NPROBE_PARAMETER: &str = "ivf.nprobe";
-const EF_SEARCH_PARAMETER: &str = "hnsw.ef_search";
pub struct VindexVectorGlobalIndexReader {
io_meta: GlobalIndexIOMeta,
@@ -152,22 +150,15 @@ fn search_vindex(
return Ok(None);
}
- let params = VectorSearchParams::with_ef_search(
- effective_k,
- int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE)?,
- int_parameter(options, EF_SEARCH_PARAMETER, DEFAULT_EF_SEARCH)?,
- );
+ let nprobe = int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE)?;
+ let params = VectorSearchParams::new(effective_k, nprobe);
let (labels, distances) = if let Some(include_ids) =
&vector_search.include_row_ids {
if include_ids.is_empty() {
return Ok(None);
}
let ek = std::cmp::min(effective_k, include_ids.len() as usize);
- let params = VectorSearchParams::with_ef_search(
- params.top_k.min(ek),
- params.nprobe,
- params.ef_search,
- );
+ let params = VectorSearchParams::new(params.top_k.min(ek), nprobe);
let mut filter_bytes = Vec::new();
include_ids
.serialize_into(&mut filter_bytes)
@@ -323,12 +314,7 @@ mod tests {
int_parameter(&options, NPROBE_PARAMETER, DEFAULT_NPROBE).unwrap(),
32
);
- assert_eq!(
- int_parameter(&options, EF_SEARCH_PARAMETER,
DEFAULT_EF_SEARCH).unwrap(),
- DEFAULT_EF_SEARCH
- );
-
- options.insert(EF_SEARCH_PARAMETER.to_string(), "abc".to_string());
- assert!(int_parameter(&options, EF_SEARCH_PARAMETER,
DEFAULT_EF_SEARCH).is_err());
+ options.insert(NPROBE_PARAMETER.to_string(), "abc".to_string());
+ assert!(int_parameter(&options, NPROBE_PARAMETER,
DEFAULT_NPROBE).is_err());
}
}
diff --git a/docs/src/sql.md b/docs/src/sql.md
index 42c78cbb..b0da06bf 100644
--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -963,8 +963,8 @@ CREATE TABLE paimon.my_db.items (
);
```
-For vector indexes backed by vindex, set `index_type` to one of `ivf-flat`,
-`ivf-pq`, `ivf-hnsw-flat`, or `ivf-hnsw-sq`:
+For vector indexes backed by vindex, set `index_type` to `ivf-flat` or
+`ivf-pq`:
```sql
CALL sys.create_global_index(
@@ -1003,12 +1003,9 @@ Supported vindex options:
| `<index-type>.nlist` | `256` | all vindex types | Number of IVF lists. |
| `<index-type>.pq.m` | `16` | `ivf-pq` | Number of product-quantization
sub-vectors. The dimension must be divisible by this value. |
| `<index-type>.pq.use-opq` | `false` | `ivf-pq` | Whether to enable OPQ
before PQ encoding. |
-| `<index-type>.hnsw.m` | native default | HNSW vindex types | HNSW graph
connectivity. |
-| `<index-type>.hnsw.ef-construction` | native default | HNSW vindex types |
HNSW construction beam width. |
-| `<index-type>.hnsw.max-level` | native default | HNSW vindex types | Maximum
HNSW graph level. |
Native vindex aliases are also accepted in the `options` string: `dimension`,
-`metric`, `nlist`, `pq.m`, `use-opq`, and `hnsw.*`.
+`metric`, `nlist`, `pq.m`, and `use-opq`.
Inspect committed index files with the `$table_indexes` system table:
