This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new eb6a4f30 fix(lumina): reject non-finite vector elements when building
an index (#898)
eb6a4f30 is described below
commit eb6a4f3071e92038971a3d696905e85971eff162
Author: jackylee <[email protected]>
AuthorDate: Tue Sep 22 14:14:53 2026 +0800
fix(lumina): reject non-finite vector elements when building an index (#898)
---
.../table/lumina_index_build_builder/extraction.rs | 18 +++++++-
.../src/table/lumina_index_build_builder/tests.rs | 52 ++++++++++++++++++++++
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs
b/crates/paimon/src/table/lumina_index_build_builder/extraction.rs
index 1076507a..7fb5b10f 100644
--- a/crates/paimon/src/table/lumina_index_build_builder/extraction.rs
+++ b/crates/paimon/src/table/lumina_index_build_builder/extraction.rs
@@ -181,7 +181,23 @@ pub(super) fn extract_vectors_from_batches(
source: None,
});
}
- vectors.push(values.value(value_index));
+ let value = values.value(value_index);
+ // Java rejects these per element in
`LuminaVectorGlobalIndexWriter`, and
+ // so does the vindex backend (`paimon_vindex_core`'s
+ // `validate_finite_values`). Without the check the index is
built from
+ // undefined data and every later top-k over the shard is
scored against
+ // it.
+ if !value.is_finite() {
+ return Err(Error::DataInvalid {
+ message: format!(
+ "Lumina vector extraction found non-finite element
{value} \
+ at _ROW_ID {row_id}, index {}",
+ value_index - start
+ ),
+ source: None,
+ });
+ }
+ vectors.push(value);
}
}
}
diff --git a/crates/paimon/src/table/lumina_index_build_builder/tests.rs
b/crates/paimon/src/table/lumina_index_build_builder/tests.rs
index 48351c8f..b4e59878 100644
--- a/crates/paimon/src/table/lumina_index_build_builder/tests.rs
+++ b/crates/paimon/src/table/lumina_index_build_builder/tests.rs
@@ -555,6 +555,25 @@ fn test_extract_vectors_rejects_non_list_float32() {
assert!(matches!(err, Error::DataInvalid { message, .. } if
message.contains("List<Float32>")));
}
+#[test]
+fn test_extract_vectors_rejects_non_finite_element() {
+ for value in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] {
+ let batch = vector_batch(vec![Some(vec![Some(1.0), Some(value)])],
vec![Some(7)]);
+
+ let err = extract_vectors_from_batches(&[batch], "embedding", 2, 7, 1)
+ .expect_err("a non-finite element should fail");
+
+ let Error::DataInvalid { message, .. } = err else {
+ panic!("expected DataInvalid for {value}, got {err:?}");
+ };
+ assert!(message.contains("non-finite"), "message: {message}");
+ // The row id and the position inside the vector are what make the
failure
+ // actionable, as in Java's `Vector element at rowId=%d, index=%d`.
+ assert!(message.contains("_ROW_ID 7"), "message: {message}");
+ assert!(message.contains("index 1"), "message: {message}");
+ }
+}
+
fn fixed_size_vector_batch(
rows: Vec<Option<Vec<f32>>>,
row_ids: Vec<Option<i64>>,
@@ -673,6 +692,39 @@ fn
test_extract_vectors_fixed_size_list_rejects_null_element() {
);
}
+#[test]
+fn test_extract_vectors_fixed_size_list_rejects_non_finite_element() {
+ // Both layouts share the element loop, so this pins that the FixedSizeList
+ // path reaches it too.
+ let element_field = Arc::new(ArrowField::new("element",
ArrowDataType::Float32, true));
+ let mut builder = FixedSizeListBuilder::new(Float32Builder::new(),
2).with_field(element_field);
+ builder.values().append_value(1.0);
+ builder.values().append_value(f32::NAN);
+ builder.append(true);
+ let row_ids = Arc::new(Int64Array::from(vec![Some(0)])) as ArrayRef;
+ let schema = Arc::new(ArrowSchema::new(vec![
+ ArrowField::new(
+ "embedding",
+ ArrowDataType::FixedSizeList(
+ Arc::new(ArrowField::new("element", ArrowDataType::Float32,
true)),
+ 2,
+ ),
+ true,
+ ),
+ ArrowField::new(ROW_ID_FIELD_NAME, ArrowDataType::Int64, true),
+ ]));
+ let batch = RecordBatch::try_new(
+ schema,
+ vec![Arc::new(builder.finish()) as ArrayRef, row_ids],
+ )
+ .unwrap();
+
+ let err = extract_vectors_from_batches(&[batch], "embedding", 2, 0, 1)
+ .expect_err("a non-finite element should fail");
+
+ assert!(matches!(err, Error::DataInvalid { message, .. } if
message.contains("non-finite")));
+}
+
#[test]
fn test_checked_metadata_conversion_supports_i64_file_size() {
let above_i32_max = i32::MAX as u64 + 1;
