This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 324e51193d perf: Count digits to skip precision check when parsing
decimals (#10998)
324e51193d is described below
commit 324e51193d610cc851a96e55a913a80a808d4828
Author: Neil Conway <[email protected]>
AuthorDate: Sat Sep 5 22:26:14 2026 -0400
perf: Count digits to skip precision check when parsing decimals (#10998)
# Which issue does this PR close?
- N/A
# Rationale for this change
`parse_decimal` validated the parsed value against the precision with
`is_valid_decimal_precision`, an out-of-line call that loads the bounds
for the precision and compares the value against both.
The parser already knows how many digits it kept. We can modify the
mantissa scan to return the digit count and then use this count to skip
the precision check for the common case that the number of parsed digits
implies the parsed value is well within the allowed precision.
`parse_decimal` microbenchmark, Apple M4 Max, µs per 1024 inputs:
```
case before after change
123.123 7.93 7.85 -1.0%
123.1234 11.39 10.96 -3.7%
123.1 11.22 10.27 -8.5%
123 10.49 9.47 -9.7%
-123.123 8.16 8.07 -1.0%
-123.1234 11.47 11.09 -3.3%
-123.1 11.45 10.53 -8.0%
-123 10.71 9.84 -8.1%
0.0000123 8.37 7.89 -5.7%
12. 10.21 9.54 -6.5%
-12. 10.44 9.76 -6.5%
00.1 10.69 9.90 -7.4%
-00.1 10.93 10.00 -8.5%
12345678912345678.1234 24.49 24.53 +0.1%
-12345678912345678.1234 24.67 24.77 +0.4%
99999999999999999.999 20.31 19.90 -2.0%
-99999999999999999.999 20.25 20.40 +0.7%
.123 6.34 6.18 -2.6%
-.123 6.59 6.41 -2.7%
123. 10.67 10.12 -5.1%
-123. 10.94 10.30 -5.8%
string decimal128 short 8.20 7.85 -4.2%
string decimal128 integer 11.19 10.89 -2.7%
string decimal128 exact scale 11.43 11.05 -3.3%
string decimal128 padded scale 10.50 10.42 -0.8%
string decimal128 rounded scale 15.23 14.63 -3.9%
string decimal128 signed 15.28 14.78 -3.3%
string decimal128 38 digits 24.19 23.65 -2.2%
string decimal128 exponent 20.28 20.18 -0.5%
string decimal128 negative exponent 19.52 18.79 -3.7%
string decimal128 negative scale 18.32 17.97 -1.9%
string decimal128 long fraction 22.39 22.14 -1.1%
string decimal256 76 digits 59.03 57.76 -2.1%
string decimal256 rounded scale 55.72 54.71 -1.8%
string decimal32 short 8.51 7.74 -9.1%
string decimal32 9 digits 10.45 9.76 -6.6%
string decimal64 short 7.29 6.84 -6.2%
string decimal64 18 digits 12.04 11.66 -3.2%
```
# What changes are included in this PR?
See above.
# Are these changes tested?
Yes; new test added, existing tests pass.
# Are there any user-facing changes?
No.
# AI usage
Developed with Claude Code Fable 5.1; reviewed with Codex Astra 6. I
reviewed, revised, and understand the resulting code.
---
arrow-cast/src/parse.rs | 104 +++++++++++++++++++++++++++++++++++++-----------
1 file changed, 81 insertions(+), 23 deletions(-)
diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs
index d077a9c5a4..ffc6738aa9 100644
--- a/arrow-cast/src/parse.rs
+++ b/arrow-cast/src/parse.rs
@@ -861,8 +861,12 @@ pub(crate) fn parse_decimal_checked<T: DecimalType>(
precision: u8,
scale: i8,
) -> Result<T::Native, DecimalParseError> {
- let value = parse_decimal_native::<T>(s, scale)?;
- if T::is_valid_decimal_precision(value, precision) {
+ let (value, digits) = parse_decimal_native::<T>(s, scale)?;
+ // A value of at most `precision` digits is within the precision without
+ // inspecting it. A precision beyond the type's maximum is invalid.
+ let fits = precision <= T::MAX_PRECISION
+ && (digits <= precision as usize ||
T::is_valid_decimal_precision(value, precision));
+ if fits {
Ok(value)
} else {
Err(DecimalParseError::Overflow)
@@ -870,20 +874,22 @@ pub(crate) fn parse_decimal_checked<T: DecimalType>(
}
/// Parses `s` as a decimal with the given `scale` into the native type of `T`,
-/// checking only that the result fits the native type (not the precision).
+/// checking only that the result fits the native type (not the precision),
+/// and returns it with an upper bound on its number of decimal digits.
///
/// See [`parse_decimal`] for the accepted syntax and rounding behaviour.
+#[inline]
fn parse_decimal_native<T: DecimalType>(
s: &str,
scale: i8,
-) -> Result<T::Native, DecimalParseError> {
+) -> Result<(T::Native, usize), DecimalParseError> {
let bytes = s.as_bytes().trim_ascii();
let (negative, mut mantissa) = split_sign(bytes);
let mut scale = scale as i64;
loop {
let exponent_at = match parse_decimal_mantissa::<T>(mantissa,
negative, scale) {
- Ok(value) => return Ok(value),
+ Ok(result) => return Ok(result),
Err(MantissaError::InvalidFormat) => return
Err(DecimalParseError::InvalidFormat),
Err(MantissaError::Exponent(index)) => index,
// The digits before an exponent marker need not fit on their own
@@ -934,13 +940,15 @@ const MAX_CHUNK_DIGITS: usize = 18;
/// Scans `mantissa` (digits with at most one decimal point; the sign has
/// already been removed) and folds the digits that are significant at
/// `scale` into a native value, rounding half away from zero on the first
-/// digit that is not.
+/// digit that is not. Also returns an upper bound on the number of decimal
+/// digits of the value: the digits kept, the zeros appended to reach the
+/// scale, and the digit that rounding up can add.
#[inline]
fn parse_decimal_mantissa<T: DecimalType>(
mantissa: &[u8],
negative: bool,
scale: i64,
-) -> Result<T::Native, MantissaError> {
+) -> Result<(T::Native, usize), MantissaError> {
// The number of integer and fractional digits that contribute to the
// result. For a non-negative scale that is every integer digit and the
// first `scale` fractional digits. For a negative scale the last `-scale`
@@ -1036,7 +1044,10 @@ fn parse_decimal_mantissa<T: DecimalType>(
.map_err(|_| MantissaError::Overflow)?;
}
- Ok(value)
+ let digits = usize::try_from(missing.max(0))
+ .unwrap_or(usize::MAX)
+ .saturating_add(int_kept + frac_kept + round as usize);
+ Ok((value, digits))
}
/// Parses the digits of an exponent (`[+|-] digits`), saturating at the bounds
@@ -1705,6 +1716,11 @@ mod tests {
use arrow_array::temporal_conversions::date32_to_datetime;
use arrow_buffer::i256;
+ /// Parses `s` without a precision check, for probing the native range
+ fn parse_native<T: DecimalType>(s: &str, scale: i8) -> Result<T::Native,
DecimalParseError> {
+ parse_decimal_native::<T>(s, scale).map(|(value, _)| value)
+ }
+
#[test]
fn test_parse_nanos() {
assert_eq!(parse_nanos::<3, 0>(&[1, 2, 3]), 123_000_000);
@@ -3106,65 +3122,107 @@ mod tests {
// ... or past the native type itself
assert_eq!(
- parse_decimal_native::<Decimal32Type>("2147483647.5", 0),
+ parse_native::<Decimal32Type>("2147483647.5", 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal32Type>("-2147483648.5", 0),
+ parse_native::<Decimal32Type>("-2147483648.5", 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal128Type>(&format!("{}.5",
i128::MAX), 0),
+ parse_native::<Decimal128Type>(&format!("{}.5", i128::MAX), 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal128Type>(&format!("{}.5",
i128::MIN), 0),
+ parse_native::<Decimal128Type>(&format!("{}.5", i128::MIN), 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal256Type>(&format!("{}.5",
i256::MAX), 0),
+ parse_native::<Decimal256Type>(&format!("{}.5", i256::MAX), 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal256Type>(&format!("{}.5",
i256::MIN), 0),
+ parse_native::<Decimal256Type>(&format!("{}.5", i256::MIN), 0),
Err(DecimalParseError::Overflow)
);
}
+ #[test]
+ fn test_parse_decimal_precision_by_digit_count() {
+ // Rounding up can add a digit
+ assert_eq!(
+ parse_decimal::<Decimal128Type>("99999.4", 5, 0).unwrap(),
+ 99999
+ );
+ assert!(parse_decimal::<Decimal128Type>("99999.5", 5, 0).is_err());
+ assert!(parse_decimal::<Decimal128Type>("-99999.5", 5, 0).is_err());
+ assert_eq!(
+ parse_decimal::<Decimal128Type>("99999.5", 6, 0).unwrap(),
+ 100000
+ );
+ // Leading zeros count as digits only for the shortcut; the value is
+ // then checked by its range
+ assert_eq!(
+ parse_decimal::<Decimal128Type>("000000000000000000000001", 1,
0).unwrap(),
+ 1
+ );
+ assert_eq!(
+ parse_decimal::<Decimal128Type>("0.000000000000000000001", 1,
21).unwrap(),
+ 1
+ );
+ assert!(parse_decimal::<Decimal128Type>("0.0000000000000000000012", 1,
22).is_err());
+ // The zeros appended to reach the scale count as digits
+ assert_eq!(parse_decimal::<Decimal128Type>("1", 3, 2).unwrap(), 100);
+ assert!(parse_decimal::<Decimal128Type>("1", 2, 2).is_err());
+ assert!(parse_decimal::<Decimal128Type>("1e2", 2, 0).is_err());
+ assert_eq!(parse_decimal::<Decimal32Type>("1e2", 3, 0).unwrap(), 100);
+ // Scaling down leaves fewer digits
+ assert_eq!(
+ parse_decimal::<Decimal128Type>("123456", 2, -4).unwrap(),
+ 12
+ );
+ assert!(parse_decimal::<Decimal128Type>("123456", 1, -4).is_err());
+ // A precision beyond the type's maximum is invalid
+ assert!(parse_decimal::<Decimal32Type>("1", 10, 0).is_err());
+ assert!(parse_decimal::<Decimal32Type>("00000000001", 10, 0).is_err());
+ assert!(parse_decimal::<Decimal128Type>("1", 39, 0).is_err());
+ assert!(parse_decimal::<Decimal256Type>("1", 77, 0).is_err());
+ }
+
#[test]
fn test_parse_decimal_native_full_range() {
// The native range exceeds the largest precision; the precision check
// is the caller's responsibility
assert_eq!(
- parse_decimal_native::<Decimal32Type>("-2147483648", 0),
+ parse_native::<Decimal32Type>("-2147483648", 0),
Ok(i32::MIN)
);
assert_eq!(
- parse_decimal_native::<Decimal32Type>("2147483648", 0),
+ parse_native::<Decimal32Type>("2147483648", 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal64Type>("-9223372036854775808", 0),
+ parse_native::<Decimal64Type>("-9223372036854775808", 0),
Ok(i64::MIN)
);
assert_eq!(
- parse_decimal_native::<Decimal64Type>("9223372036854775808", 0),
+ parse_native::<Decimal64Type>("9223372036854775808", 0),
Err(DecimalParseError::Overflow)
);
assert_eq!(
- parse_decimal_native::<Decimal128Type>(&i128::MAX.to_string(), 0),
+ parse_native::<Decimal128Type>(&i128::MAX.to_string(), 0),
Ok(i128::MAX)
);
assert_eq!(
- parse_decimal_native::<Decimal128Type>(&i128::MIN.to_string(), 0),
+ parse_native::<Decimal128Type>(&i128::MIN.to_string(), 0),
Ok(i128::MIN)
);
assert_eq!(
- parse_decimal_native::<Decimal256Type>(&i256::MAX.to_string(), 0),
+ parse_native::<Decimal256Type>(&i256::MAX.to_string(), 0),
Ok(i256::MAX)
);
assert_eq!(
- parse_decimal_native::<Decimal256Type>(&i256::MIN.to_string(), 0),
+ parse_native::<Decimal256Type>(&i256::MIN.to_string(), 0),
Ok(i256::MIN)
);
// The unscaled value (integer digits scaled by 10^21) far exceeds the
@@ -3172,7 +3230,7 @@ mod tests {
// arbitrary (possibly in-range) value
let input = format!("{}.12345678901234567890123", "7".repeat(71));
assert_eq!(
- parse_decimal_native::<Decimal256Type>(&input, 21),
+ parse_native::<Decimal256Type>(&input, 21),
Err(DecimalParseError::Overflow)
);
