This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11187-61c69cd16dcc9dc15a109adbacba479b8d08d9f5 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 646328f240763b2ac161b86d60b99d8d4324af68 Author: Neil Conway <[email protected]> AuthorDate: Sat Oct 3 16:16:28 2026 +0000 perf: Optimize `date_part` for timestamp values without timezones (#11187) # Which issue does this PR close? - N/A # Rationale for this change For timestamp inputs, `date_part` generally converts the timestamp to a Chrono value and then uses Chrono to extract the requested component. For timestamps without a timezone, hour through nanosecond components can be calculated directly from the stored integer, without determining the calendar date. This is roughly 5-20x faster than going through Chrono on a microbenchmark (results below). This is partly motivated by improving the performance of `date_part` and `extract` in DataFusion. For example, ClickBench Q18 improves by about 2% using this optimization. Benchmarks: (ARM64, elapsed times in microseconds) - timestamp_s/Hour/no_nulls: 33.366 → 5.827, −82.5% - timestamp_s/Minute/no_nulls: 36.901 → 7.108, −80.7% - timestamp_s/Second/no_nulls: 35.321 → 2.530, −92.8% - timestamp_s/Hour/mixed_nulls: 30.885 → 5.821, −81.2% - timestamp_s/Year/control: 32.893 → 32.913, +0.1% - timestamp_s/Minute/timezone_control: 59.305 → 59.081, −0.4% - timestamp_s/Second/timezone_control: 55.782 → 56.263, +0.9% - timestamp_ms/Hour/no_nulls: 45.351 → 5.839, −87.1% - timestamp_ms/Minute/no_nulls: 49.214 → 6.235, −87.3% - timestamp_ms/Second/no_nulls: 47.252 → 6.226, −86.8% - timestamp_ms/Millisecond/no_nulls: 44.510 → 2.317, −94.8% - timestamp_ms/Microsecond/no_nulls: 44.503 → 2.472, −94.4% - timestamp_ms/Nanosecond/no_nulls: 43.116 → 2.375, −94.5% - timestamp_ms/Hour/mixed_nulls: 39.190 → 5.859, −85.1% - timestamp_ms/Year/control: 44.099 → 44.754, +1.5% - timestamp_ms/Minute/timezone_control: 73.426 → 73.751, +0.4% - timestamp_ms/Nanosecond/timezone_control: 65.517 → 65.771, +0.4% - timestamp_us/Hour/no_nulls: 41.028 → 6.520, −84.1% - timestamp_us/Minute/no_nulls: 44.774 → 6.777, −84.9% - timestamp_us/Second/no_nulls: 42.947 → 6.238, −85.5% - timestamp_us/Millisecond/no_nulls: 40.022 → 2.629, −93.4% - timestamp_us/Microsecond/no_nulls: 39.975 → 2.304, −94.2% - timestamp_us/Nanosecond/no_nulls: 39.116 → 2.365, −94.0% - timestamp_us/Hour/mixed_nulls: 39.438 → 6.486, −83.6% - timestamp_us/Year/control: 39.921 → 40.406, +1.2% - timestamp_us/Minute/timezone_control: 68.042 → 68.354, +0.5% - timestamp_us/Nanosecond/timezone_control: 61.106 → 61.602, +0.8% - timestamp_ns/Hour/no_nulls: 40.454 → 3.540, −91.2% - timestamp_ns/Minute/no_nulls: 44.499 → 3.388, −92.4% - timestamp_ns/Second/no_nulls: 42.425 → 6.241, −85.3% - timestamp_ns/Millisecond/no_nulls: 39.546 → 2.628, −93.4% - timestamp_ns/Microsecond/no_nulls: 39.388 → 2.627, −93.3% - timestamp_ns/Nanosecond/no_nulls: 38.727 → 2.304, −94.1% - timestamp_ns/Hour/mixed_nulls: 38.805 → 3.542, −90.9% - timestamp_ns/Year/control: 39.567 → 39.710, +0.4% - timestamp_ns/Minute/timezone_control: 69.019 → 68.191, −1.2% - timestamp_ns/Nanosecond/timezone_control: 61.682 → 61.172, −0.8% # What changes are included in this PR? * Optimize `date_part` as described above * Add unit tests * Add benchmark # Are these changes tested? Existing tests pass. Two new tests were added: one checks the consistency of this code path with the results produced by Chrono; the second checks the results of this code path for extreme values (outside of Chrono's supported calendar range). # Are there any user-facing changes? There is one user-visible behavior change. Chrono has a limited calendar range (roughly +/- 262,000 years); calling `date_part` on timestamps outside that range returned null. This implementation is defined for the entire range of timestamp values. The returned value is correct, but this behavior is slightly inconsistent with `date_part` for units that are still implemented via Chrono, and for timestamp values with time zones. # AI usage Developed with Codex Astra 6, reviewed with Claude Code Fable 5.1. I then reviewed and revised the resulting code. --------- Co-authored-by: Jeffrey Vo <[email protected]> --- arrow-arith/Cargo.toml | 4 + arrow-arith/benches/temporal.rs | 91 ++++++++++++++++++++ arrow-arith/src/temporal.rs | 184 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+) diff --git a/arrow-arith/Cargo.toml b/arrow-arith/Cargo.toml index 4f15871f65..8bb02b1d8c 100644 --- a/arrow-arith/Cargo.toml +++ b/arrow-arith/Cargo.toml @@ -49,5 +49,9 @@ criterion = { workspace = true } name = "decimal_arithmetic" harness = false +[[bench]] +name = "temporal" +harness = false + [lints] workspace = true diff --git a/arrow-arith/benches/temporal.rs b/arrow-arith/benches/temporal.rs new file mode 100644 index 0000000000..e2861a7097 --- /dev/null +++ b/arrow-arith/benches/temporal.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hint::black_box; + +use arrow_arith::temporal::{DatePart, date_part}; +use arrow_array::PrimitiveArray; +use arrow_array::types::*; +use criterion::{Criterion, Throughput, criterion_group, criterion_main}; + +const SIZE: usize = 8192; + +const TIME_PARTS: [DatePart; 6] = [ + DatePart::Hour, + DatePart::Minute, + DatePart::Second, + DatePart::Millisecond, + DatePart::Microsecond, + DatePart::Nanosecond, +]; + +fn benchmark<T: ArrowTimestampType>(c: &mut Criterion, unit: &str, units_per_second: i64) { + // A deterministic mix of dates before and after the epoch, including subseconds. + let values: Vec<i64> = (0..SIZE) + .map(|i| { + let seconds = (i as i64 * 1_000_003) % 4_000_000_000 - 2_000_000_000; + seconds * units_per_second + (i as i64 * 7919) % units_per_second + }) + .collect(); + let with_nulls = PrimitiveArray::<T>::from_iter( + values + .iter() + .enumerate() + .map(|(i, &v)| (i % 5 != 0).then_some(v)), + ); + let array = PrimitiveArray::<T>::new(values.into(), None); + let mut group = c.benchmark_group(format!("timestamp_{unit}")); + group.throughput(Throughput::Elements(SIZE as u64)); + // Second timestamps have no fractional part, so their subsecond parts are + // constant zero and not worth measuring. + let parts = if units_per_second == 1 { + &TIME_PARTS[..3] + } else { + &TIME_PARTS[..] + }; + for &part in parts { + group.bench_function(format!("{part}/no_nulls"), |b| { + b.iter(|| black_box(date_part(black_box(&array), part).unwrap())) + }); + } + // Validity is not consulted per element, so one nullable case suffices. + group.bench_function("Hour/mixed_nulls", |b| { + b.iter(|| black_box(date_part(black_box(&with_nulls), DatePart::Hour).unwrap())) + }); + // Reference for the calendar conversion path. + group.bench_function("Year/control", |b| { + b.iter(|| black_box(date_part(black_box(&array), DatePart::Year).unwrap())) + }); + // Timestamps with a timezone always take the calendar conversion path. + let with_timezone = array.with_timezone("+05:45"); + for part in [DatePart::Minute, parts[parts.len() - 1]] { + group.bench_function(format!("{part}/timezone_control"), |b| { + b.iter(|| black_box(date_part(black_box(&with_timezone), part).unwrap())) + }); + } + group.finish(); +} + +fn temporal(c: &mut Criterion) { + benchmark::<TimestampSecondType>(c, "s", 1); + benchmark::<TimestampMillisecondType>(c, "ms", 1_000); + benchmark::<TimestampMicrosecondType>(c, "us", 1_000_000); + benchmark::<TimestampNanosecondType>(c, "ns", 1_000_000_000); +} + +criterion_group!(benches, temporal); +criterion_main!(benches); diff --git a/arrow-arith/src/temporal.rs b/arrow-arith/src/temporal.rs index b4653c2fe7..f42d99d4c3 100644 --- a/arrow-arith/src/temporal.rs +++ b/arrow-arith/src/temporal.rs @@ -184,6 +184,10 @@ where /// Returns an [`Int32Array`] unless input was a dictionary type, in which case returns /// the dictionary but with this function applied onto its values. /// +/// Null inputs produce null outputs. A timestamp outside the supported calendar +/// range also produces null, except that for timestamps without a timezone the +/// time parts (`Hour` through `Nanosecond`) are defined for every value. +/// /// If array passed in is not of the above listed types (or is a dictionary array where the /// values array isn't of the above listed types), then this function will return an error. /// @@ -415,6 +419,62 @@ impl ExtractDatePartExt for PrimitiveArray<Date64Type> { } } +/// Number of ticks in one second for timestamps of type `T`. +const fn units_per_second<T: ArrowTimestampType>() -> i64 { + match T::UNIT { + TimeUnit::Second => 1, + TimeUnit::Millisecond => MILLISECONDS, + TimeUnit::Microsecond => MICROSECONDS, + TimeUnit::Nanosecond => NANOSECONDS, + } +} + +/// Extracts a time-of-day part from a timestamp array without a timezone using +/// integer arithmetic. Returns `None` for parts that require a calendar +/// conversion. +/// +/// `div_euclid` rounds toward negative infinity and `rem_euclid` yields a +/// nonnegative remainder, so pre-epoch values produce in-range components: +/// -1 ms is 1969-12-31 23:59:59.999, whose second is 59 and millisecond is 999. +fn timestamp_time_part<T: ArrowTimestampType>( + array: &PrimitiveArray<T>, + part: DatePart, +) -> Option<Int32Array> { + Some(match part { + DatePart::Hour => { + array.unary(|v| v.div_euclid(units_per_second::<T>() * 3_600).rem_euclid(24) as i32) + } + DatePart::Minute => { + array.unary(|v| v.div_euclid(units_per_second::<T>() * 60).rem_euclid(60) as i32) + } + DatePart::Second => { + array.unary(|v| v.div_euclid(units_per_second::<T>()).rem_euclid(60) as i32) + } + DatePart::Millisecond => { + array.unary(|v| subsecond(v, units_per_second::<T>(), MILLISECONDS)) + } + DatePart::Microsecond => { + array.unary(|v| subsecond(v, units_per_second::<T>(), MICROSECONDS)) + } + DatePart::Nanosecond => array.unary(|v| subsecond(v, units_per_second::<T>(), NANOSECONDS)), + _ => return None, + }) +} + +/// Fractional second of `v`, counted in `units` per second, rescaled to `target` +/// per second. Both scales divide one second evenly, so the conversion is exact +/// when scaling up and truncating when scaling down. +#[inline] +fn subsecond(v: i64, units: i64, target: i64) -> i32 { + let fraction = v.rem_euclid(units); + let scaled = if target >= units { + fraction * (target / units) + } else { + fraction / (units / target) + }; + scaled as i32 +} + impl ExtractDatePartExt for PrimitiveArray<TimestampSecondType> { fn date_part(&self, part: DatePart) -> Result<Int32Array, ArrowError> { // TimestampSecond only encodes number of seconds, so these will always be 0 @@ -428,6 +488,8 @@ impl ExtractDatePartExt for PrimitiveArray<TimestampSecondType> { .map(|c| Utc.from_utc_datetime(&c).with_timezone(&tz)) .map(map_func) }) + } else if let Some(array) = timestamp_time_part(self, part) { + array } else { let map_func = get_date_time_part_extract_fn(part); self.unary_opt(|d| timestamp_s_to_datetime(d).map(map_func)) @@ -445,6 +507,8 @@ impl ExtractDatePartExt for PrimitiveArray<TimestampMillisecondType> { .map(|c| Utc.from_utc_datetime(&c).with_timezone(&tz)) .map(map_func) }) + } else if let Some(array) = timestamp_time_part(self, part) { + array } else { let map_func = get_date_time_part_extract_fn(part); self.unary_opt(|d| timestamp_ms_to_datetime(d).map(map_func)) @@ -462,6 +526,8 @@ impl ExtractDatePartExt for PrimitiveArray<TimestampMicrosecondType> { .map(|c| Utc.from_utc_datetime(&c).with_timezone(&tz)) .map(map_func) }) + } else if let Some(array) = timestamp_time_part(self, part) { + array } else { let map_func = get_date_time_part_extract_fn(part); self.unary_opt(|d| timestamp_us_to_datetime(d).map(map_func)) @@ -479,6 +545,8 @@ impl ExtractDatePartExt for PrimitiveArray<TimestampNanosecondType> { .map(|c| Utc.from_utc_datetime(&c).with_timezone(&tz)) .map(map_func) }) + } else if let Some(array) = timestamp_time_part(self, part) { + array } else { let map_func = get_date_time_part_extract_fn(part); self.unary_opt(|d| timestamp_ns_to_datetime(d).map(map_func)) @@ -752,6 +820,9 @@ impl<T: Datelike> ChronoDateExt for T { mod tests { use super::*; + use arrow_array::temporal_conversions::as_datetime; + use chrono::NaiveDate; + /// Used to integrate new [`date_part()`] method with deprecated shims such as /// [`hour()`] and [`week()`]. fn date_part_primitive<T: ArrowTemporalType>( @@ -762,6 +833,119 @@ mod tests { Ok(array.as_primitive::<Int32Type>().to_owned()) } + /// Time-of-day parts of a timestamp. + const TIME_PARTS: [DatePart; 6] = [ + DatePart::Hour, + DatePart::Minute, + DatePart::Second, + DatePart::Millisecond, + DatePart::Microsecond, + DatePart::Nanosecond, + ]; + + /// Checks time part extraction against the calendar conversion for values within + /// its supported range. Covers component boundaries, negative timestamps, nulls + /// and slices, plus timezone dispatch and invalid timezone metadata. + fn check_timestamp_time_parts<T: ArrowTimestampType>(units_per_second: i64) { + let mut values = vec![i64::MIN, i64::MAX]; + for boundary in [1_000, 1_000_000] { + for delta in [-1, 0, 1] { + values.extend([boundary + delta, -boundary + delta]); + } + } + for base in [0, 60, 3_600, 86_400] { + for delta in [-1, 0, 1] { + for seconds in [base + delta, -base + delta] { + for fraction in [-1, 0, 1, units_per_second - 1] { + let value = i128::from(seconds) * i128::from(units_per_second) + + i128::from(fraction); + if let Ok(value) = i64::try_from(value) { + values.push(value); + } + } + } + } + } + // Sample both the full i64 domain and dates near the epoch: most full-range + // second and millisecond timestamps are outside the supported calendar range. + let mut seed = 42_u64; + for _ in 0..256 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + values.push(seed as i64); + let seconds = (seed % 4_000_000_000) as i64 - 2_000_000_000; + values.push(seconds * units_per_second + (seed % units_per_second as u64) as i64); + } + // The calendar conversion provides expected values only within its range. + // Behavior at the i64 extremes is checked separately. + values.retain(|&v| as_datetime::<T>(v).is_some()); + let no_nulls = PrimitiveArray::<T>::new(values.clone().into(), None); + let mixed = PrimitiveArray::<T>::from_iter( + values + .iter() + .enumerate() + .map(|(i, &v)| (i % 5 != 0).then_some(v)), + ); + for array in [no_nulls, mixed.slice(1, mixed.len() - 2)] { + for part in TIME_PARTS { + let map_func = get_date_time_part_extract_fn(part); + let expected: Int32Array = array.unary_opt(|v| as_datetime::<T>(v).map(map_func)); + let actual = date_part(&array, part).unwrap(); + assert_eq!( + actual.as_primitive::<Int32Type>(), + &expected, + "{} {part}", + T::DATA_TYPE + ); + } + } + + // An offset that changes both hour and minute checks that timezone metadata + // still routes through the calendar conversion. + let zoned = PrimitiveArray::<T>::from_iter_values([0]).with_timezone("+05:45"); + for (part, expected) in [(DatePart::Hour, 5), (DatePart::Minute, 45)] { + assert_eq!( + date_part_primitive(&zoned, part).unwrap(), + Int32Array::from(vec![expected]) + ); + } + let invalid = PrimitiveArray::<T>::from_iter_values([0]).with_timezone("invalid timezone"); + assert!(date_part(&invalid, DatePart::Second).is_err()); + } + + #[test] + fn test_timestamp_time_parts_match_chrono() { + check_timestamp_time_parts::<TimestampSecondType>(1); + check_timestamp_time_parts::<TimestampMillisecondType>(MILLISECONDS); + check_timestamp_time_parts::<TimestampMicrosecondType>(MICROSECONDS); + check_timestamp_time_parts::<TimestampNanosecondType>(NANOSECONDS); + } + + #[test] + fn test_timestamp_time_parts_extremes() { + /// `(hour, minute, second, nanosecond)` expected at `i64::MIN` and `i64::MAX`. + fn check<T: ArrowTimestampType>(min: (u32, u32, u32, u32), max: (u32, u32, u32, u32)) { + let time = |(h, m, s, ns)| NaiveDate::default().and_hms_nano_opt(h, m, s, ns).unwrap(); + let expected = [Some(time(min)), None, Some(time(max))]; + let array = PrimitiveArray::<T>::from_iter([Some(i64::MIN), None, Some(i64::MAX)]); + for part in TIME_PARTS { + let map_func = get_date_time_part_extract_fn(part); + assert_eq!( + date_part_primitive(&array, part).unwrap(), + Int32Array::from_iter(expected.iter().map(|t| t.map(map_func))), + "{} {part}", + T::DATA_TYPE + ); + } + } + + // These values lie outside the supported calendar range for these units, so + // the expectations cannot be derived from the calendar conversion. Nanosecond + // extremes are within range and are covered by the comparison against it. + check::<TimestampSecondType>((8, 29, 52, 0), (15, 30, 7, 0)); + check::<TimestampMillisecondType>((16, 47, 4, 192_000_000), (7, 12, 55, 807_000_000)); + check::<TimestampMicrosecondType>((19, 59, 5, 224_192_000), (4, 0, 54, 775_807_000)); + } + #[test] fn test_temporal_array_date64_hour() { let a: PrimitiveArray<Date64Type> =
