mbutrovich commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4605619220
##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday,
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64`
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+ ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+ (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) ->
Int32Array {
+ if dates.null_count() * 2 <= dates.len() {
+ dates.unary::<_, Int32Type>(kernel)
+ } else {
+ dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+ }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ arg: ColumnarValue,
+ kernel: F,
+) -> Result<ColumnarValue> {
+ match arg {
+ ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+ }
+ ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+ }
+ ColumnarValue::Scalar(scalar) => {
+ let array = scalar.to_array_of_size(1)?;
+ let result = epoch_day_array(name, &array, kernel)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ }
+ ColumnarValue::Array(array) => {
+ Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+ }
+ }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ array: &ArrayRef,
+ kernel: F,
+) -> Result<ArrayRef> {
+ // A dictionary-encoded date column: map the dictionary values, rewrap the
+ // keys, then unpack to a plain Int32 array, so the work is proportional to
+ // the cardinality rather than the row count.
+ if let DataType::Dictionary(_, _) = array.data_type() {
+ let dict = array.as_any_dictionary();
+ let Some(values) = dict.values().as_primitive_opt::<Date32Type>() else
{
+ return exec_err!("`{name}` expects a date, got {}",
array.data_type());
+ };
+ let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef;
+ return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?);
+ }
+ let Some(dates) = array.as_primitive_opt::<Date32Type>() else {
Review Comment:
> Both `.slt` files cover a scalar, a column with NULL and a date before the
epoch, and `dayofweek.slt` also covers a dictionary.
The dictionary query in `dayofweek.slt` doesn't reach the new
`DataType::Date64` arm in the dictionary branch ([`dayofweek.rs`
L97-L100](https://github.com/apache/datafusion-spark/blob/be65c39bc4a2defadb120cb3892c61032333eb62/crates/spark/src/function/datetime/dayofweek.rs#L97-L100)).
I replaced that arm with `DataType::Date64 => Arc::clone(dict.values())`, and
`dayofweek.slt` still passed. Coercion casts the dictionary to its value type
before the function runs. With a column input, `EXPLAIN` shows
`dayofweek(CAST(CAST(CAST(column1 AS Date64) AS Dictionary(Int32, Date64)) AS
Date64))`, and the all-literal query in the file is folded to `Int32(1)` at
planning.
Comet calls `invoke_with_args` directly, so only a unit test reaches this
arm, the way `dictionary_input_unpacks` does for `Date32`
([L286-L299](https://github.com/apache/datafusion-spark/blob/be65c39bc4a2defadb120cb3892c61032333eb62/crates/spark/src/function/datetime/dayofweek.rs#L286-L299)).
Could we add one next to it, and drop the dictionary query from
`dayofweek.slt`? For example, with `Date64Array` added to the `use
arrow::array::DictionaryArray;` import:
```rust
#[test]
fn date64_dictionary_input() {
let values =
Arc::new(Date64Array::from(vec![Some(0), Some(-86_400_000), None]))
as ArrayRef;
let keys = Int32Array::from(vec![Some(0), Some(1), None, Some(2)]);
let dict = DictionaryArray::<Int32Type>::new(keys, values);
let out = invoke_array(Arc::new(dict));
assert_eq!(out.iter().collect::<Vec<_>>(), [Some(5), Some(4), None,
None]);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]