This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch 59_maintenance
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/59_maintenance by this push:
new 1bbe4caa82 [59_maintenance] fix(arrow-data): fix gaps in ListView
equality (#11439)
1bbe4caa82 is described below
commit 1bbe4caa826f026809f3a4ca441371a31cd23a0c
Author: Jeffrey Vo <[email protected]>
AuthorDate: Fri Oct 9 02:52:35 2026 +0900
[59_maintenance] fix(arrow-data): fix gaps in ListView equality (#11439)
- Backport https://github.com/apache/arrow-rs/pull/11345
- For inclusion in https://github.com/apache/arrow-rs/issues/11352
Cherry-pick of
https://github.com/apache/arrow-rs/commit/862b1e065b602400784268df9ded5d0a18710eed
---------
Co-authored-by: Neil Conway <[email protected]>
---
Cargo.lock | 9 +++++----
arrow-data/src/equal/list_view.rs | 23 +++++++++++++++--------
arrow-data/src/equal/mod.rs | 3 +++
arrow/tests/array_equal.rs | 26 ++++++++++++++++++++++++++
4 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index f70f848d43..65f3d138c7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -604,9 +604,9 @@ checksum =
"c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-lc-rs"
-version = "1.16.3"
+version = "1.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -614,14 +614,15 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.40.0"
+version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
+ "pkg-config",
]
[[package]]
diff --git a/arrow-data/src/equal/list_view.rs
b/arrow-data/src/equal/list_view.rs
index c7cb31db90..299414b135 100644
--- a/arrow-data/src/equal/list_view.rs
+++ b/arrow-data/src/equal/list_view.rs
@@ -17,7 +17,7 @@
use crate::ArrayData;
use crate::data::count_nulls;
-use crate::equal::equal_values;
+use crate::equal::equal_range;
use arrow_buffer::ArrowNativeType;
use num_integer::Integer;
@@ -74,8 +74,7 @@ pub(super) fn list_view_equal<T: ArrowNativeType + Integer>(
let rhs_offset = rhs_offset.to_usize().unwrap();
let size = size.to_usize().unwrap();
- // Check if offsets are valid for the given range
- if !equal_values(lhs_data, rhs_data, lhs_offset, rhs_offset, size)
{
+ if !equal_range(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
return false;
}
}
@@ -101,10 +100,10 @@ pub(super) fn list_view_equal<T: ArrowNativeType +
Integer>(
return false;
}
- for (index, ((&lhs_offset, &rhs_offset), &size)) in lhs_range_offsets
+ for (index, ((&lhs_offset, &rhs_offset), (&lhs_size, &rhs_size))) in
lhs_range_offsets
.iter()
.zip(rhs_range_offsets)
- .zip(lhs_range_sizes)
+ .zip(lhs_range_sizes.iter().zip(rhs_range_sizes))
.enumerate()
{
let lhs_is_null = lhs_nulls.is_null(index);
@@ -114,12 +113,20 @@ pub(super) fn list_view_equal<T: ArrowNativeType +
Integer>(
return false;
}
+ // Null rows are equal whatever their sizes and values
+ if lhs_is_null {
+ continue;
+ }
+
+ if lhs_size != rhs_size {
+ return false;
+ }
+
let lhs_offset = lhs_offset.to_usize().unwrap();
let rhs_offset = rhs_offset.to_usize().unwrap();
- let size = size.to_usize().unwrap();
+ let size = lhs_size.to_usize().unwrap();
- // Check if values match in the range
- if !lhs_is_null && !equal_values(lhs_data, rhs_data, lhs_offset,
rhs_offset, size) {
+ if !equal_range(lhs_data, rhs_data, lhs_offset, rhs_offset, size) {
return false;
}
}
diff --git a/arrow-data/src/equal/mod.rs b/arrow-data/src/equal/mod.rs
index b241b8cf66..011f591d10 100644
--- a/arrow-data/src/equal/mod.rs
+++ b/arrow-data/src/equal/mod.rs
@@ -58,6 +58,9 @@ use variable_size::variable_sized_equal;
/// Compares the values of two [ArrayData] starting at `lhs_start` and
`rhs_start` respectively
/// for `len` slots.
+///
+/// Assumes the null masks of the two ranges are already known to be equal. To
compare
+/// ranges of child data, use [`equal_range`], which also compares the null
masks.
#[inline]
fn equal_values(
lhs: &ArrayData,
diff --git a/arrow/tests/array_equal.rs b/arrow/tests/array_equal.rs
index 381054a25d..c21096c6dc 100644
--- a/arrow/tests/array_equal.rs
+++ b/arrow/tests/array_equal.rs
@@ -822,6 +822,22 @@ fn test_test_list_view_array<T: OffsetSizeTrait>() {
test_equal(&a, &b, true);
}
+fn test_list_view_sizes_and_child_nulls<T: OffsetSizeTrait>() {
+ // Same first elements but different sizes, with a null row
+ let a = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(2)]),
None]);
+ let b = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(2),
Some(3)]), None]);
+ test_equal(&a, &b, false);
+
+ // Child values that differ only in their null masks, without and with a
null row
+ let a = create_list_view_array::<T, _, _>([Some(vec![Some(1), Some(0)])]);
+ let b = create_list_view_array::<T, _, _>([Some(vec![Some(1), None])]);
+ test_equal(&a, &b, false);
+
+ let a = create_list_view_array::<T, _, _>([None, Some(vec![Some(1),
Some(0)])]);
+ let b = create_list_view_array::<T, _, _>([None, Some(vec![Some(1),
None])]);
+ test_equal(&a, &b, false);
+}
+
// Special test for List<ListView<i32>>.
// This tests the equal_ranges kernel
fn test_sliced_list_of_list_view<T: OffsetSizeTrait>() {
@@ -871,6 +887,16 @@ fn test_large_list_view_array() {
test_test_list_view_array::<i64>();
}
+#[test]
+fn test_list_view_array_sizes_and_child_nulls() {
+ test_list_view_sizes_and_child_nulls::<i32>();
+}
+
+#[test]
+fn test_large_list_view_array_sizes_and_child_nulls() {
+ test_list_view_sizes_and_child_nulls::<i64>();
+}
+
#[test]
fn test_nested_list_view_array() {
test_sliced_list_of_list_view::<i32>();
