mbutrovich commented on code in PR #12062:
URL: https://github.com/apache/arrow-rs/pull/12062#discussion_r4635054358


##########
arrow-arith/src/numeric.rs:
##########
@@ -1812,6 +2164,335 @@ mod tests {
         assert_eq!(err, "Divide by zero error");
     }
 
+    type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+    type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+    fn assert_decimal_result<T: DecimalType>(
+        actual: Result<ArrayRef, ArrowError>,
+        expected: Result<Vec<Option<T::Native>>, ArrowError>,
+    ) {
+        match (actual, expected) {
+            (Ok(actual), Ok(expected)) => {
+                let actual: Vec<_> = 
actual.as_primitive::<T>().iter().collect();
+                assert_eq!(actual, expected)
+            }
+            (Err(actual), Err(expected)) => assert_eq!(actual.to_string(), 
expected.to_string()),
+            (actual, expected) => panic!("got {actual:?}, expected 
{expected:?}"),
+        }
+    }
+
+    /// Checks `kernel` against `checked` for every pair of `edges`, with
+    /// arrays on both sides, with each pair as a one-row slice, and with a
+    /// scalar on either side. Every third row of the left array is null, so
+    /// null slots also hold values that overflow.
+    fn test_checked_decimal_op<T: DecimalType>(
+        edges: &[T::Native],
+        (l_scale, r_scale): (i8, i8),
+        kernel: Kernel,
+        checked: impl Fn(T::Native, T::Native) -> Result<T::Native, 
ArrowError>,
+    ) {
+        let decimal = |array: PrimitiveArray<T>, scale| {
+            array
+                .with_precision_and_scale(T::MAX_PRECISION, scale)
+                .unwrap()
+        };
+        let (l, r): (Vec<_>, Vec<_>) = edges
+            .iter()
+            .flat_map(|&x| edges.iter().map(move |&y| (x, y)))
+            .unzip();
+        let nulls = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 != 2));
+        let l_array = decimal(
+            PrimitiveArray::new(l.clone().into(), Some(nulls.clone())),
+            l_scale,
+        );
+        let r_array = 
decimal(PrimitiveArray::from_iter_values(r.iter().copied()), r_scale);
+        let row = |i: usize, x, y| match nulls.is_valid(i) {
+            true => checked(x, y).map(Some),
+            false => Ok(None),
+        };
+
+        // The kernels report the first valid row that overflows.
+        let expected = (0..l.len()).map(|i| row(i, l[i], r[i])).collect();
+        assert_decimal_result::<T>(kernel(&l_array, &r_array), expected);
+
+        // An overflow in one row sends the whole array to the checked kernel,
+        // which would hide a missed overflow in another row, so also check 
each
+        // row alone.
+        for i in 0..l.len() {
+            let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i, 1));
+            assert_decimal_result::<T>(actual, row(i, l[i], r[i]).map(|v| 
vec![v]));
+        }
+
+        for &x in edges {
+            let l_scalar = 
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), l_scale));
+            let expected = r.iter().map(|&y| checked(x, 
y).map(Some)).collect();
+            assert_decimal_result::<T>(kernel(&l_scalar, &r_array), expected);
+            let r_scalar = 
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), r_scale));
+            let expected = (0..l.len()).map(|i| row(i, l[i], x)).collect();
+            assert_decimal_result::<T>(kernel(&l_array, &r_scalar), expected);
+        }
+    }
+
+    /// Checks `add`, `sub` and `mul` of `T` at its edge values, with equal
+    /// scales and with either side rescaled by 100.
+    fn test_checked_decimal_ops<T: DecimalType>() {
+        let one = T::Native::ONE;
+        let three = T::Native::usize_as(3);
+        let two = T::Native::usize_as(2);
+        let hundred = T::Native::usize_as(100);
+        let (min, max) = (T::Native::MIN_TOTAL_ORDER, 
T::Native::MAX_TOTAL_ORDER);
+        let edges = [
+            min,
+            min.add_wrapping(one),
+            min.div_wrapping(two),
+            min.div_wrapping(hundred),
+            min.div_wrapping(hundred).sub_wrapping(one),
+            three.neg_wrapping(),
+            one.neg_wrapping(),
+            T::Native::ZERO,
+            one,
+            three,
+            max.div_wrapping(hundred),
+            max.div_wrapping(hundred).add_wrapping(one),
+            max.div_wrapping(two),
+            max.div_wrapping(two).add_wrapping(one),
+            max.sub_wrapping(one),
+            max,
+        ];
+
+        for scales in [(2, 2), (2, 4), (4, 2)] {
+            let (l_mul, r_mul) = match scales {
+                (2, 4) => (hundred, one),
+                (4, 2) => (one, hundred),
+                _ => (one, one),
+            };
+            let rescale = move |l: T::Native, r: T::Native| {
+                Ok::<_, ArrowError>((l.mul_checked(l_mul)?, 
r.mul_checked(r_mul)?))
+            };
+            test_checked_decimal_op::<T>(&edges, scales, add, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.add_checked(r)
+            });
+            test_checked_decimal_op::<T>(&edges, scales, sub, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.sub_checked(r)
+            });
+            test_checked_decimal_op::<T>(&edges, scales, add_wrapping, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.add_checked(r)
+            });
+        }
+        // Multiplication doesn't rescale its operands.
+        test_checked_decimal_op::<T>(&edges, (2, 2), mul, |l, r| 
l.mul_checked(r));
+        test_checked_decimal_op::<T>(&edges, (2, 4), mul_wrapping, |l, r| 
l.mul_checked(r));
+    }
+
+    #[test]
+    fn test_checked_decimal_edges() {
+        test_checked_decimal_ops::<Decimal32Type>();
+        test_checked_decimal_ops::<Decimal64Type>();
+        test_checked_decimal_ops::<Decimal128Type>();
+        test_checked_decimal_ops::<Decimal256Type>();
+    }
+
+    #[test]
+    fn test_checked_decimal128_mul_i64_boundary() {
+        // `i128` multiplies through `i64` when both operands fit, so check 
both
+        // sides of each `i64` bound. `i64::MIN * i64::MIN` is 2^126.
+        let (min, max) = (i64::MIN as i128, i64::MAX as i128);
+        let edges = [min - 1, min, min + 1, -1, 0, 1, max - 1, max, max + 1];
+        test_checked_decimal_op::<Decimal128Type>(&edges, (2, 2), mul, |l, r| 
l.mul_checked(r));
+        test_checked_decimal_op::<Decimal128Type>(&edges, (2, 4), add, |l, r| {
+            l.mul_checked(100)?.add_checked(r)
+        });
+    }
+
+    /// Moves one row that overflows through a batch, so that each lane of the
+    /// vector loop and each row of the scalar remainder holds it once, with an
+    /// array and a scalar on either side. The rescaled cases overflow only in
+    /// the rescale. The same row is also checked as a null slot, which is not
+    /// an error.
+    fn test_checked_decimal_overflow_position<T: DecimalType>() {
+        // Three iterations of the widest vector loop we build, 8 `i32` lanes 
of
+        // AVX2 unrolled four times, plus a remainder.
+        const LEN: usize = 100;
+        let one = T::Native::ONE;
+        let two = T::Native::usize_as(2);
+        let hundred = T::Native::usize_as(100);
+        let (min, max) = (T::Native::MIN_TOTAL_ORDER, 
T::Native::MAX_TOTAL_ORDER);
+        // Multiplying by 100 overflows, but adding or subtracting 1 doesn't.
+        let rescale_overflow = max.div_wrapping(hundred).add_wrapping(one);
+        // The kernel, its checked scalar operation, whether it rescales, the
+        // scales of the batch and the other operand, the value that overflows,
+        // and the other operand.
+        type Case<N> = (Kernel, Checked<N>, bool, (i8, i8), N, N);
+        let cases: [Case<T::Native>; 5] = [
+            (add, |l, r| l.add_checked(r), true, (2, 2), max, one),
+            (sub, |l, r| l.sub_checked(r), true, (2, 2), min, one),
+            (mul, |l, r| l.mul_checked(r), false, (2, 2), max, two),
+            (
+                add,
+                |l, r| l.add_checked(r),
+                true,
+                (2, 4),
+                rescale_overflow,
+                one,
+            ),
+            (
+                sub,
+                |l, r| l.sub_checked(r),
+                true,
+                (2, 4),
+                rescale_overflow,
+                one,
+            ),
+        ];
+        let decimal = |array: PrimitiveArray<T>, scale| {
+            array
+                .with_precision_and_scale(T::MAX_PRECISION, scale)
+                .unwrap()
+        };
+        for (kernel, checked, rescales, (batch_scale, other_scale), edge, 
other) in cases {
+            // The checked result of `l op r`, rescaling the operand with the 
smaller scale.
+            let row = |l: T::Native, l_scale, r: T::Native, r_scale| {
+                if !rescales {
+                    return checked(l, r);
+                }
+                let l = if l_scale < r_scale {
+                    l.mul_checked(hundred)?
+                } else {
+                    l
+                };
+                let r = if r_scale < l_scale {
+                    r.mul_checked(hundred)?
+                } else {
+                    r
+                };
+                checked(l, r)
+            };
+            let other_array = decimal(PrimitiveArray::from_value(other, LEN), 
other_scale);
+            let other_scalar =
+                Scalar::new(decimal(PrimitiveArray::from_value(other, 1), 
other_scale));
+            for i in 0..LEN {
+                let mut values = vec![one; LEN];
+                values[i] = edge;
+                for null in [None, Some(i)] {
+                    let nulls = null.map(|i| 
NullBuffer::from_iter((0..LEN).map(|j| j != i)));
+                    let array = decimal(
+                        PrimitiveArray::new(values.clone().into(), nulls),
+                        batch_scale,
+                    );
+                    let expected =
+                        |batch_left: bool| -> Result<Vec<Option<T::Native>>, 
ArrowError> {
+                            (0..LEN)
+                                .map(|j| match (Some(j) == null, batch_left) {
+                                    (true, _) => Ok(None),
+                                    (false, true) => {
+                                        row(values[j], batch_scale, other, 
other_scale).map(Some)
+                                    }
+                                    (false, false) => {
+                                        row(other, other_scale, values[j], 
batch_scale).map(Some)
+                                    }
+                                })
+                                .collect()
+                        };
+                    assert_decimal_result::<T>(kernel(&array, &other_array), 
expected(true));
+                    assert_decimal_result::<T>(kernel(&other_array, &array), 
expected(false));
+                    assert_decimal_result::<T>(kernel(&array, &other_scalar), 
expected(true));
+                    assert_decimal_result::<T>(kernel(&other_scalar, &array), 
expected(false));
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_checked_decimal_null_slot_overflow_sliced() {

Review Comment:
   > Could it be dropped when rebasing?
   
   I'd keep it. It's the only test that runs `decimal_op` on a sliced array 
whose null slots overflow. Until #12058 lands, it's also the only test in this 
PR of `any_valid` on a sliced validity buffer, which I added for 
https://github.com/apache/arrow-rs/pull/12062#discussion_r4579103663.
   
   > The `Kernel` and `Checked` aliases would also collide
   
   Yes. Whichever PR lands second drops its copy of the aliases along with its 
copy of the shared helpers, as in 
https://github.com/apache/arrow-rs/pull/12062#discussion_r4568419018 and 
https://github.com/apache/arrow-rs/pull/12058#discussion_r4588942444.
   
   > `test_checked_decimal_op` and `test_checked_integer_op` are close enough 
that one helper taking a closure for the data type could serve both.
   
   They differ in more than the data type. The decimal helper takes the two 
scales and checks each operation against the rescaled operands, and the integer 
helper also runs every pair without nulls. A shared helper would need both of 
those as parameters, so I'd keep them separate.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to