This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 9271c51e Fix partition field mapping for vector search source plans
(#1211)
9271c51e is described below
commit 9271c51e1f1e7e65cbeb1cdb2779767953f70639
Author: Jingsong Lee <[email protected]>
AuthorDate: Sat Oct 24 08:53:08 2026 +0800
Fix partition field mapping for vector search source plans (#1211)
---
bindings/python/tests/test_vector_search.py | 25 +++++++++++++++++++++++++
crates/paimon/src/table/de_vector_read.rs | 20 ++++++++++++++++++++
2 files changed, 45 insertions(+)
diff --git a/bindings/python/tests/test_vector_search.py
b/bindings/python/tests/test_vector_search.py
index 23762d68..70b77cfc 100644
--- a/bindings/python/tests/test_vector_search.py
+++ b/bindings/python/tests/test_vector_search.py
@@ -67,6 +67,31 @@ def
test_filters_accumulate_and_partition_uses_table_schema(table):
result.splits()
[email protected]('mode', ['fast', 'full', 'detail'])
[email protected]('batch', [False, True])
+def test_indexed_partition_filter_uses_table_field_positions(table, mode,
batch):
+ (table.new_global_index_build_builder().with_index_column('embedding')
+ .with_index_type('ivf-flat').with_options({
+ 'ivf-flat.dimension': '2', 'ivf-flat.nlist': '1',
+ 'ivf-flat.distance.metric': 'l2'}).execute())
+ (table.new_global_index_build_builder().with_index_column('id')
+ .with_index_type('btree').execute())
+ table = table.copy({'vector-index.search-mode': mode,
+ 'global-index.filter.refine-from-data': 'false'})
+ builder =
(table.new_batch_vector_search_builder().with_vector_column('embedding')
+ .with_query_vectors([[1, 0], [1, 0]]).with_limit(3) if batch
else _single(table))
+ # pt is field 2 in the table and field 0 in the partition row. An exact
+ # scalar lookup must retain the matching indexed vector in either API.
+ builder.with_filter({'method': 'equal', 'field': 'id', 'literals': [3]})
+ builder.with_partition_filter({'method': 'equal', 'field': 'pt',
'literals': [1]})
+ results = builder.execute() if batch else [builder.execute()]
+ for result in results:
+ assert len(result) == 1
+ assert list(result.row_ids().values()) == [1.0]
+ rows =
pa.Table.from_batches(result.new_read_builder().with_projection(['id']).read())
+ assert rows.column('id').to_pylist() == [3]
+
+
@pytest.mark.parametrize('batch', [False, True])
def test_partition_filter_rejects_data_fields(table, batch):
builder = table.new_batch_vector_search_builder() if batch else
table.new_vector_search_builder()
diff --git a/crates/paimon/src/table/de_vector_read.rs
b/crates/paimon/src/table/de_vector_read.rs
index 912f8045..1704da8e 100644
--- a/crates/paimon/src/table/de_vector_read.rs
+++ b/crates/paimon/src/table/de_vector_read.rs
@@ -782,6 +782,26 @@ async fn evaluate_batch_vector_search_with_executor(
table.schema().partition_keys(),
);
if let Some(partition) = partition {
+ // The split helper projects onto the partition row, while the
+ // source scan expects positions in the complete table schema.
+ let mapping = table
+ .schema()
+ .partition_keys()
+ .iter()
+ .map(|name| {
+ table
+ .schema()
+ .fields()
+ .iter()
+ .position(|field| field.name() == name)
+ })
+ .collect::<Vec<_>>();
+ let partition =
partition.remap_field_index(&mapping).ok_or_else(|| {
+ crate::Error::DataInvalid {
+ message: "Cannot map source partition filter to table
fields".into(),
+ source: None,
+ }
+ })?;
let partition_ids = Arc::new(
super::search_scan_split::partition_row_ids_for_search_sources(
table,
