alamb commented on code in PR #8711:
URL: https://github.com/apache/arrow-rs/pull/8711#discussion_r2466821877
##########
arrow-select/src/zip.rs:
##########
@@ -279,4 +294,110 @@ mod test {
let expected = Int32Array::from(vec![None, None, Some(42), Some(42),
None]);
assert_eq!(actual, &expected);
}
+
+ #[test]
+ fn
test_zip_kernel_primitive_scalar_with_boolean_array_mask_with_nulls_should_be_treated_as_false()
+ {
+ let scalar_truthy = Scalar::new(Int32Array::from_value(42, 1));
+ let scalar_falsy = Scalar::new(Int32Array::from_value(123, 1));
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_any().downcast_ref::<Int32Array>().unwrap();
+ let expected = Int32Array::from(vec![
+ Some(42),
+ Some(42),
+ Some(123),
+ Some(123), // true in mask but null
+ Some(123),
+ Some(123),
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn
test_zip_primitive_array_with_nulls_is_mask_should_be_treated_as_false() {
+ let scalar_truthy = Int32Array::from_iter_values(vec![1, 2, 3, 4, 5,
6]);
+ let scalar_falsy = Int32Array::from_iter_values(vec![7, 8, 9, 10, 11,
12]);
Review Comment:
nit is that these are not scalars (they are arrays)
```suggestion
let truthy = Int32Array::from_iter_values(vec![1, 2, 3, 4, 5, 6]);
let falsy = Int32Array::from_iter_values(vec![7, 8, 9, 10, 11, 12]);
```
##########
arrow-select/src/zip.rs:
##########
@@ -279,4 +294,110 @@ mod test {
let expected = Int32Array::from(vec![None, None, Some(42), Some(42),
None]);
assert_eq!(actual, &expected);
}
+
+ #[test]
+ fn
test_zip_kernel_primitive_scalar_with_boolean_array_mask_with_nulls_should_be_treated_as_false()
+ {
+ let scalar_truthy = Scalar::new(Int32Array::from_value(42, 1));
+ let scalar_falsy = Scalar::new(Int32Array::from_value(123, 1));
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_any().downcast_ref::<Int32Array>().unwrap();
+ let expected = Int32Array::from(vec![
+ Some(42),
+ Some(42),
+ Some(123),
+ Some(123), // true in mask but null
+ Some(123),
+ Some(123),
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn
test_zip_primitive_array_with_nulls_is_mask_should_be_treated_as_false() {
+ let scalar_truthy = Int32Array::from_iter_values(vec![1, 2, 3, 4, 5,
6]);
+ let scalar_falsy = Int32Array::from_iter_values(vec![7, 8, 9, 10, 11,
12]);
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_any().downcast_ref::<Int32Array>().unwrap();
+ let expected = Int32Array::from(vec![
+ Some(1),
+ Some(2),
+ Some(9),
+ Some(10), // true in mask but null
+ Some(11),
+ Some(12),
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn test_zip_string_array_with_nulls_is_mask_should_be_treated_as_false() {
+ let scalar_truthy = StringArray::from_iter_values(vec!["1", "2", "3",
"4", "5", "6"]);
Review Comment:
ditto here, these are not scalar
##########
arrow-select/src/zip.rs:
##########
@@ -279,4 +294,110 @@ mod test {
let expected = Int32Array::from(vec![None, None, Some(42), Some(42),
None]);
assert_eq!(actual, &expected);
}
+
+ #[test]
+ fn
test_zip_kernel_primitive_scalar_with_boolean_array_mask_with_nulls_should_be_treated_as_false()
+ {
+ let scalar_truthy = Scalar::new(Int32Array::from_value(42, 1));
+ let scalar_falsy = Scalar::new(Int32Array::from_value(123, 1));
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_any().downcast_ref::<Int32Array>().unwrap();
+ let expected = Int32Array::from(vec![
+ Some(42),
+ Some(42),
+ Some(123),
+ Some(123), // true in mask but null
+ Some(123),
+ Some(123),
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn
test_zip_primitive_array_with_nulls_is_mask_should_be_treated_as_false() {
+ let scalar_truthy = Int32Array::from_iter_values(vec![1, 2, 3, 4, 5,
6]);
+ let scalar_falsy = Int32Array::from_iter_values(vec![7, 8, 9, 10, 11,
12]);
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_any().downcast_ref::<Int32Array>().unwrap();
+ let expected = Int32Array::from(vec![
+ Some(1),
+ Some(2),
+ Some(9),
+ Some(10), // true in mask but null
+ Some(11),
+ Some(12),
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn test_zip_string_array_with_nulls_is_mask_should_be_treated_as_false() {
+ let scalar_truthy = StringArray::from_iter_values(vec!["1", "2", "3",
"4", "5", "6"]);
+ let scalar_falsy = StringArray::from_iter_values(vec!["7", "8", "9",
"10", "11", "12"]);
+
+ let mask = {
+ let booleans = BooleanBuffer::from(vec![true, true, false, true,
false, false]);
+ let nulls = NullBuffer::from(vec![
+ true, true, true,
+ false, // null treated as false even though in the original
mask it was true
+ true, true,
+ ]);
+ BooleanArray::new(booleans, Some(nulls))
+ };
+ let out = zip(&mask, &scalar_truthy, &scalar_falsy).unwrap();
+ let actual = out.as_string::<i32>();
+ let expected = StringArray::from_iter_values(vec![
+ "1", "2", "9", "10", // true in mask but null
+ "11", "12",
+ ]);
+ assert_eq!(actual, &expected);
+ }
+
+ #[test]
+ fn
test_zip_kernel_large_string_scalar_with_boolean_array_mask_with_nulls_should_be_treated_as_false()
Review Comment:
another nit is that the paths are reversed -- for primitives it tests
scalars first then arrays but for strings it tests arrays first then strings
##########
arrow-select/src/zip.rs:
##########
@@ -166,9 +168,22 @@ pub fn zip(
Ok(make_array(data))
}
+fn maybe_prep_null_mask_filter(predicate: &BooleanArray) -> BooleanBuffer {
+ // Nulls are treated as false
Review Comment:
as a follow on PR this would be a nice improvement to make to
`prep_null_mask_filter` itself -- handling arrays without nulls
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
