sunchao commented on code in PR #12062:
URL: https://github.com/apache/arrow-rs/pull/12062#discussion_r4577272471
##########
arrow-arith/src/numeric.rs:
##########
@@ -327,6 +328,299 @@ macro_rules! try_op_ref {
}};
}
+/// Wrapping `+`, `-` and `*` that also report whether they overflowed, written
+/// so that LLVM vectorizes a loop over them.
+///
+/// The signed `overflowing_add` and `overflowing_sub` lower to
+/// `llvm.sadd.with.overflow` and `llvm.ssub.with.overflow`, which LLVM doesn't
+/// vectorize, so the signed impls test the sign bits instead.
+trait OverflowingOp: Sized {
+ /// Whether the checked kernels use the one-pass path for this type. When
+ /// it's false, [`overflowing_op`] returns `None` without doing any work.
+ const ONE_PASS: bool = true;
+
+ fn add_overflowing(self, rhs: Self) -> (Self, bool);
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool);
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool);
+}
+
+macro_rules! signed_overflowing_op {
+ ($($t:ty $(=> $wide:ty)?),*) => {$(
+ impl OverflowingOp for $t {
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ let sum = self.wrapping_add(rhs);
+ // Overflow iff both operands have the same sign and `sum` has
the other.
+ (sum, (self ^ sum) & (rhs ^ sum) < 0)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ let difference = self.wrapping_sub(rhs);
+ // Overflow iff the operands have different signs and
`difference`
+ // doesn't have the sign of `self`.
+ (difference, (self ^ rhs) & (self ^ difference) < 0)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ signed_mul_overflowing!(self, rhs, $t $(, $wide)?)
+ }
+ }
+ )*};
+}
+
+/// The wrapping product of two signed integers and whether it overflowed.
+macro_rules! signed_mul_overflowing {
+ // The truncating cast is the wrapping product. It overflowed iff widening
+ // it back doesn't give the full product.
+ ($l:expr, $r:expr, $t:ty, $wide:ty) => {{
+ let product = <$wide>::from($l) * <$wide>::from($r);
+ (product as $t, <$wide>::from(product as $t) != product)
+ }};
+ // Widened to `i128`, an `i64` multiply stays a scalar `imul` on x86-64,
and
+ // LLVM packs each row's overflow flag into vector registers around it,
+ // which made the loop slower than the checked kernel with
+ // `-C target-cpu=x86-64-v3`.
+ ($l:expr, $r:expr, $t:ty) => {
+ $l.overflowing_mul($r)
+ };
+}
+
+signed_overflowing_op!(i8 => i16, i16 => i32, i32 => i64, i64);
+
+macro_rules! unsigned_overflowing_op {
+ ($($t:ty),*) => {$(
+ // LLVM vectorizes the unsigned `overflowing_add` and
`overflowing_sub`.
+ // It folds a widening unsigned multiply back into
`llvm.umul.with.overflow`,
+ // which it doesn't vectorize, so `mul` stays scalar either way.
+ impl OverflowingOp for $t {
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_add(rhs)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_sub(rhs)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_mul(rhs)
+ }
+ }
+ )*};
+}
+
+unsigned_overflowing_op!(u8, u16, u32, u64);
+
+/// Applies `op` to every slot of `l` and `r`, including null slots, without
+/// branching on overflow.
+///
+/// Returns `Ok(None)` if a valid row overflowed or the array lengths differ,
+/// so the caller can fall back to the checked kernel to report the error.
+fn overflowing_op<T, F>(
+ l: &PrimitiveArray<T>,
+ l_s: bool,
+ r: &PrimitiveArray<T>,
+ r_s: bool,
+ op: F,
+) -> Result<Option<PrimitiveArray<T>>, ArrowError>
+where
+ T: ArrowPrimitiveType,
+ T::Native: OverflowingOp,
+ F: Fn(T::Native, T::Native) -> (T::Native, bool),
+{
+ if !T::Native::ONE_PASS {
+ return Ok(None);
+ }
+ match (l_s, r_s) {
+ (true, true) | (false, false) => {
+ if l.len() != r.len() {
+ return Ok(None);
+ }
+ let pairs = l.values().iter().zip(r.values());
+ let values = collect_overflowing(pairs.map(|(&l, &r)| op(l, r)))?;
+ let nulls = NullBuffer::union(l.nulls(), r.nulls());
+ let (l, r) = (l.values(), r.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r[i]).1))
+ }
+ (true, false) => match (l.null_count() == 0).then(|| l.value(0)) {
+ None => Ok(Some(PrimitiveArray::new_null(r.len()))),
+ Some(l) => {
+ let values = collect_overflowing(r.values().iter().map(|&r|
op(l, r)))?;
+ let (nulls, r) = (r.nulls().cloned(), r.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l, r[i]).1))
+ }
+ },
+ (false, true) => match (r.null_count() == 0).then(|| r.value(0)) {
+ None => Ok(Some(PrimitiveArray::new_null(l.len()))),
+ Some(r) => {
+ let values = collect_overflowing(l.values().iter().map(|&l|
op(l, r)))?;
+ let (nulls, l) = (l.nulls().cloned(), l.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r).1))
+ }
+ },
+ }
+}
+
+/// Builds the array from the output of [`collect_overflowing`], or returns
+/// `None` if a valid row overflowed.
+///
+/// A null slot can hold any value, such as the 0 that builders write, which
+/// overflows unsigned `0 - 1`. So when a slot overflowed, this recomputes the
+/// overflow of every row with `overflowed` and masks it with the validity
+/// bits, instead of making the caller rerun the whole batch.
+fn unless_valid_overflow<T: ArrowPrimitiveType>(
+ (values, overflow): (Vec<T::Native>, bool),
+ nulls: Option<NullBuffer>,
+ overflowed: impl Fn(usize) -> bool,
+) -> Option<PrimitiveArray<T>> {
+ let valid_overflow = overflow
+ && nulls.as_ref().is_none_or(|n| {
+ let overflowed = BooleanBuffer::collect_bool(n.len(), overflowed);
Review Comment:
Could the validity check avoid allocating two temporary bitmaps while
holding the output? For nullable array–scalar arithmetic, the old kernel
allocated the output and cloned the input validity buffer. If only a null slot
overflows, the new check keeps the output but can panic if `collect_bool`
cannot allocate its scratch bitmap. `n.valid_indices().any(overflowed)` would
avoid both scratch buffers and short-circuit on the first valid overflow. If
bit packing is faster, `MutableBuffer::try_collect_bool` and a word-by-word
intersection would keep that loop and avoid the second allocation.
##########
arrow-arith/src/numeric.rs:
##########
@@ -327,6 +328,299 @@ macro_rules! try_op_ref {
}};
}
+/// Wrapping `+`, `-` and `*` that also report whether they overflowed, written
+/// so that LLVM vectorizes a loop over them.
+///
+/// The signed `overflowing_add` and `overflowing_sub` lower to
+/// `llvm.sadd.with.overflow` and `llvm.ssub.with.overflow`, which LLVM doesn't
+/// vectorize, so the signed impls test the sign bits instead.
+trait OverflowingOp: Sized {
+ /// Whether the checked kernels use the one-pass path for this type. When
+ /// it's false, [`overflowing_op`] returns `None` without doing any work.
+ const ONE_PASS: bool = true;
+
+ fn add_overflowing(self, rhs: Self) -> (Self, bool);
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool);
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool);
+}
+
+macro_rules! signed_overflowing_op {
+ ($($t:ty $(=> $wide:ty)?),*) => {$(
+ impl OverflowingOp for $t {
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ let sum = self.wrapping_add(rhs);
+ // Overflow iff both operands have the same sign and `sum` has
the other.
+ (sum, (self ^ sum) & (rhs ^ sum) < 0)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ let difference = self.wrapping_sub(rhs);
+ // Overflow iff the operands have different signs and
`difference`
+ // doesn't have the sign of `self`.
+ (difference, (self ^ rhs) & (self ^ difference) < 0)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ signed_mul_overflowing!(self, rhs, $t $(, $wide)?)
+ }
+ }
+ )*};
+}
+
+/// The wrapping product of two signed integers and whether it overflowed.
+macro_rules! signed_mul_overflowing {
+ // The truncating cast is the wrapping product. It overflowed iff widening
+ // it back doesn't give the full product.
+ ($l:expr, $r:expr, $t:ty, $wide:ty) => {{
+ let product = <$wide>::from($l) * <$wide>::from($r);
+ (product as $t, <$wide>::from(product as $t) != product)
+ }};
+ // Widened to `i128`, an `i64` multiply stays a scalar `imul` on x86-64,
and
+ // LLVM packs each row's overflow flag into vector registers around it,
+ // which made the loop slower than the checked kernel with
+ // `-C target-cpu=x86-64-v3`.
+ ($l:expr, $r:expr, $t:ty) => {
+ $l.overflowing_mul($r)
+ };
+}
+
+signed_overflowing_op!(i8 => i16, i16 => i32, i32 => i64, i64);
+
+macro_rules! unsigned_overflowing_op {
+ ($($t:ty),*) => {$(
+ // LLVM vectorizes the unsigned `overflowing_add` and
`overflowing_sub`.
+ // It folds a widening unsigned multiply back into
`llvm.umul.with.overflow`,
+ // which it doesn't vectorize, so `mul` stays scalar either way.
+ impl OverflowingOp for $t {
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_add(rhs)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_sub(rhs)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_mul(rhs)
+ }
+ }
+ )*};
+}
+
+unsigned_overflowing_op!(u8, u16, u32, u64);
+
+/// Applies `op` to every slot of `l` and `r`, including null slots, without
+/// branching on overflow.
+///
+/// Returns `Ok(None)` if a valid row overflowed or the array lengths differ,
+/// so the caller can fall back to the checked kernel to report the error.
+fn overflowing_op<T, F>(
+ l: &PrimitiveArray<T>,
+ l_s: bool,
+ r: &PrimitiveArray<T>,
+ r_s: bool,
+ op: F,
+) -> Result<Option<PrimitiveArray<T>>, ArrowError>
+where
+ T: ArrowPrimitiveType,
+ T::Native: OverflowingOp,
+ F: Fn(T::Native, T::Native) -> (T::Native, bool),
+{
+ if !T::Native::ONE_PASS {
+ return Ok(None);
+ }
+ match (l_s, r_s) {
+ (true, true) | (false, false) => {
+ if l.len() != r.len() {
+ return Ok(None);
+ }
+ let pairs = l.values().iter().zip(r.values());
+ let values = collect_overflowing(pairs.map(|(&l, &r)| op(l, r)))?;
+ let nulls = NullBuffer::union(l.nulls(), r.nulls());
+ let (l, r) = (l.values(), r.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r[i]).1))
+ }
+ (true, false) => match (l.null_count() == 0).then(|| l.value(0)) {
+ None => Ok(Some(PrimitiveArray::new_null(r.len()))),
+ Some(l) => {
+ let values = collect_overflowing(r.values().iter().map(|&r|
op(l, r)))?;
+ let (nulls, r) = (r.nulls().cloned(), r.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l, r[i]).1))
+ }
+ },
+ (false, true) => match (r.null_count() == 0).then(|| r.value(0)) {
+ None => Ok(Some(PrimitiveArray::new_null(l.len()))),
+ Some(r) => {
+ let values = collect_overflowing(l.values().iter().map(|&l|
op(l, r)))?;
+ let (nulls, l) = (l.nulls().cloned(), l.values());
+ Ok(unless_valid_overflow(values, nulls, |i| op(l[i], r).1))
+ }
+ },
+ }
+}
+
+/// Builds the array from the output of [`collect_overflowing`], or returns
+/// `None` if a valid row overflowed.
+///
+/// A null slot can hold any value, such as the 0 that builders write, which
+/// overflows unsigned `0 - 1`. So when a slot overflowed, this recomputes the
+/// overflow of every row with `overflowed` and masks it with the validity
+/// bits, instead of making the caller rerun the whole batch.
+fn unless_valid_overflow<T: ArrowPrimitiveType>(
+ (values, overflow): (Vec<T::Native>, bool),
+ nulls: Option<NullBuffer>,
+ overflowed: impl Fn(usize) -> bool,
+) -> Option<PrimitiveArray<T>> {
+ let valid_overflow = overflow
+ && nulls.as_ref().is_none_or(|n| {
+ let overflowed = BooleanBuffer::collect_bool(n.len(), overflowed);
+ (n.inner() & &overflowed).count_set_bits() != 0
+ });
+ (!valid_overflow).then(|| PrimitiveArray::new(values.into(), nulls))
+}
+
+/// Converts an allocation failure in [`collect_overflowing`].
+///
+/// One shared, out-of-line copy keeps the formatting code out of every
+/// instance of [`collect_overflowing`], as [`mul_checked_cold`] does for the
+/// decimal `div` and `rem` closures.
+#[cold]
+#[inline(never)]
+fn memory_error(e: TryReserveError) -> ArrowError {
+ ArrowError::MemoryError(e.to_string())
+}
+
+/// Collects the values of `iter`, and whether any was flagged as overflowed.
+///
+/// This intentional `inline(never)` keeps `overflow` in a register. Inlined
+/// into its caller under thin LTO, LLVM keeps the flag in memory and loads
+/// and stores it on every row, which serializes the loop and keeps it from
+/// vectorizing. The loop writes into the `Vec`'s spare capacity itself,
+/// because whether LLVM inlines the loop inside `collect` depends on the
+/// closure and the target, and when it doesn't, that loop reaches the flag
+/// through a pointer. The checked `add` cases of the `arithmetic_kernels`
+/// benchmark measure this.
+#[inline(never)]
+fn collect_overflowing<N>(
+ iter: impl ExactSizeIterator<Item = (N, bool)>,
+) -> Result<(Vec<N>, bool), ArrowError> {
+ let mut values = Vec::new();
+ values.try_reserve_exact(iter.len()).map_err(memory_error)?;
+ let mut overflow = false;
+ let mut written = 0;
+ for (slot, (value, overflowed)) in
values.spare_capacity_mut().iter_mut().zip(iter) {
+ slot.write(value);
+ overflow |= overflowed;
+ written += 1;
+ }
+ debug_assert!(written <= values.capacity());
+ // SAFETY: the loop initialized the first `written` slots of the spare
+ // capacity, and `written` counts those writes, so it doesn't rely on
+ // `iter.len()`.
+ unsafe { values.set_len(written) };
+ Ok((values, overflow))
+}
+
+// No vector lanes are 128 or 256 bits wide, so loops over these types stay
+// scalar. Their `overflowing_*` methods don't branch on overflow, which still
+// removes the early return of the checked kernel.
+impl OverflowingOp for i128 {
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_add(rhs)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_sub(rhs)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ // Decimal values usually fit in 64 bits, and the product of two `i64`
+ // can't overflow `i128`, so this skips the full 128-bit multiply.
+ match (i64::try_from(self), i64::try_from(rhs)) {
+ (Ok(l), Ok(r)) => (i128::from(l) * i128::from(r), false),
+ _ => self.overflowing_mul(rhs),
+ }
+ }
+}
+
+// On `x86-64-v3`, LLVM vectorizes the one-pass `i256` add loop by packing the
+// carries into vector registers, which is slower than the checked kernel.
+impl OverflowingOp for i256 {
+ const ONE_PASS: bool = false;
+
+ #[inline]
+ fn add_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_add(rhs)
+ }
+
+ #[inline]
+ fn sub_overflowing(self, rhs: Self) -> (Self, bool) {
+ self.overflowing_sub(rhs)
+ }
+
+ #[inline]
+ fn mul_overflowing(self, rhs: Self) -> (Self, bool) {
+ // `i256` has no `overflowing_mul`.
+ match self.checked_mul(rhs) {
+ Some(product) => (product, false),
+ None => (self.wrapping_mul(rhs), true),
+ }
+ }
+}
+
+/// Multiplies `l` by `mul`, then applies `op`, reporting whether either step
+/// overflowed. With unequal scales only the operand with the smaller scale is
+/// multiplied, because the other multiplier is one.
+#[inline]
+fn rescale_left<N: OverflowingOp>(l: N, mul: N, r: N, op: impl Fn(N, N) -> (N,
bool)) -> (N, bool) {
+ let (l, l_overflow) = l.mul_overflowing(mul);
+ let (value, overflow) = op(l, r);
+ (value, l_overflow | overflow)
+}
+
+/// Multiplies the value of the decimal scalar `array` by `mul`, so that the
+/// kernel doesn't rescale it on every row.
+///
+/// Returns `None` if `array` isn't a valid scalar, `mul` is one, or the
product
+/// overflows. The kernel then rescales the scalar on every row, so that an
+/// overflow is an error only if a valid row needs the value.
+fn rescale_scalar<T: ArrowPrimitiveType>(
+ array: &PrimitiveArray<T>,
+ is_scalar: bool,
+ mul: T::Native,
+) -> Option<PrimitiveArray<T>>
+where
+ T::Native: OverflowingOp,
+{
+ if !is_scalar || mul.is_eq(T::Native::ONE) || array.is_null(0) {
+ return None;
+ }
+ let (value, overflow) = array.value(0).mul_overflowing(mul);
+ (!overflow).then(|| PrimitiveArray::new(vec![value].into(), None))
Review Comment:
Could we pass the rescaled native value directly or make this temporary
reservation fallible? With two valid scalars at different scales, such as `1.50
+ 1.0000`, the [base binary
path](https://github.com/apache/arrow-rs/blob/0ada6ff1ae5f500efd51757993abbfa9fa558a53/arrow-arith/src/arity.rs#L370-L390)
returns `ArrowError::MemoryError` if output allocation fails. The new
`vec![value]` can abort first, before either the fast path's fallible
reservation or the checked fallback. This also allocates when the other operand
is an empty array and will never read the rescaled value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]