andygrove commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4607469420
##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday,
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64`
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+ ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+ (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) ->
Int32Array {
+ if dates.null_count() * 2 <= dates.len() {
+ dates.unary::<_, Int32Type>(kernel)
+ } else {
+ dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+ }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ arg: ColumnarValue,
+ kernel: F,
+) -> Result<ColumnarValue> {
+ match arg {
+ ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+ }
+ ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+ }
+ ColumnarValue::Scalar(scalar) => {
+ let array = scalar.to_array_of_size(1)?;
+ let result = epoch_day_array(name, &array, kernel)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ }
+ ColumnarValue::Array(array) => {
+ Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+ }
+ }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ array: &ArrayRef,
+ kernel: F,
+) -> Result<ArrayRef> {
+ // A dictionary-encoded date column: map the dictionary values, rewrap the
+ // keys, then unpack to a plain Int32 array, so the work is proportional to
+ // the cardinality rather than the row count.
+ if let DataType::Dictionary(_, _) = array.data_type() {
+ let dict = array.as_any_dictionary();
+ let Some(values) = dict.values().as_primitive_opt::<Date32Type>() else
{
+ return exec_err!("`{name}` expects a date, got {}",
array.data_type());
+ };
+ let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef;
+ return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?);
+ }
+ let Some(dates) = array.as_primitive_opt::<Date32Type>() else {
Review Comment:
Added `date64_dictionary_input` next to the Date32 dictionary test and
removed the Date64 dictionary query from the .slt in 3e80f6c. With the arm
replaced by `Arc::clone(dict.values())` the new test fails with "expects a
date, got Dictionary(Int32, Date64)".
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]