andygrove commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4614247870
##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday,
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64`
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+ ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+ (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) ->
Int32Array {
+ if dates.null_count() * 2 <= dates.len() {
+ dates.unary::<_, Int32Type>(kernel)
+ } else {
+ dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+ }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ arg: ColumnarValue,
+ kernel: F,
+) -> Result<ColumnarValue> {
+ match arg {
+ ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+ }
+ ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+ }
+ ColumnarValue::Scalar(scalar) => {
+ let array = scalar.to_array_of_size(1)?;
+ let result = epoch_day_array(name, &array, kernel)?;
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ }
+ ColumnarValue::Array(array) => {
+ Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+ }
+ }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+ name: &str,
+ array: &ArrayRef,
+ kernel: F,
+) -> Result<ArrayRef> {
+ // The date signature also admits Date64, which holds milliseconds since
the
+ // epoch. Arrow's cast turns it into epoch days.
+ if array.data_type() == &DataType::Date64 {
+ return epoch_day_array(name, &cast(array, &DataType::Date32)?, kernel);
Review Comment:
Fixed in ce9eb14. Date64 milliseconds are now floored to days in i64, so
values outside the Date32 range get a day of the week instead of NULL. This
also fixes times before the epoch that are not a whole day, which the cast
moved to the following day. Both .slt files now cover these values as a plain
column and as a Dictionary(Int32, Date64) column, and a unit test covers
i64::MIN and i64::MAX.
##########
crates/sqllogictest/test_files/datetime/weekday.slt:
##########
@@ -53,6 +53,13 @@ SELECT weekday('2024-01-07'::DATE);
----
6
+# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11, which
+# are outside chrono's calendar. Verified against Spark 4.2.0.
+query II
+SELECT weekday(date_from_unix_date(-2147483648)),
weekday(date_from_unix_date(2147483647));
Review Comment:
Added in ce9eb14 for both functions. The two dates also pass on Spark 4.2.0
as a column.
##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday,
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64`
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+ ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+ (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) ->
Int32Array {
Review Comment:
Switched to `unary` for every batch in ce9eb14. The kernels are total, so
evaluating the slots under NULLs is safe.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]