kosiew commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4612700261


##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDFImpl,
+    Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, 
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` 
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+    ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+    (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) -> 
Int32Array {
+    if dates.null_count() * 2 <= dates.len() {
+        dates.unary::<_, Int32Type>(kernel)
+    } else {
+        dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+    }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    arg: ColumnarValue,
+    kernel: F,
+) -> Result<ColumnarValue> {
+    match arg {
+        ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+        }
+        ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+        }
+        ColumnarValue::Scalar(scalar) => {
+            let array = scalar.to_array_of_size(1)?;
+            let result = epoch_day_array(name, &array, kernel)?;
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        }
+        ColumnarValue::Array(array) => {
+            Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+        }
+    }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    array: &ArrayRef,
+    kernel: F,
+) -> Result<ArrayRef> {
+    // The date signature also admits Date64, which holds milliseconds since 
the
+    // epoch. Arrow's cast turns it into epoch days.
+    if array.data_type() == &DataType::Date64 {
+        return epoch_day_array(name, &cast(array, &DataType::Date32)?, kernel);

Review Comment:
   Date64 values beyond the Date32 range cast to NULL, violating the 
non-nullable output contract for both UDFs. Please compute epoch days in i64 
and add non-nullable SQL projection and dictionary regression tests.



##########
crates/sqllogictest/test_files/datetime/weekday.slt:
##########
@@ -53,6 +53,13 @@ SELECT weekday('2024-01-07'::DATE);
 ----
 6
 
+# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11, which
+# are outside chrono's calendar. Verified against Spark 4.2.0.
+query II
+SELECT weekday(date_from_unix_date(-2147483648)), 
weekday(date_from_unix_date(2147483647));

Review Comment:
   Could we test both UDFs using a two-row VALUES column containing i32::MIN 
and i32::MAX? This would exercise array evaluation without constant folding.



##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDFImpl,
+    Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, 
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` 
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+    ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+    (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) -> 
Int32Array {

Review Comment:
   Could we use unary unconditionally, or benchmark the 50% NULL cutoff against 
all-valid, mixed, and mostly-NULL batches? The extra branch lacks performance 
evidence.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to