andygrove commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4603635099


##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDFImpl,
+    Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, 
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` 
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+    ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+    (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) -> 
Int32Array {
+    if dates.null_count() * 2 <= dates.len() {
+        dates.unary::<_, Int32Type>(kernel)
+    } else {
+        dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+    }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    arg: ColumnarValue,
+    kernel: F,
+) -> Result<ColumnarValue> {
+    match arg {
+        ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+        }
+        ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+        }
+        ColumnarValue::Scalar(scalar) => {
+            let array = scalar.to_array_of_size(1)?;
+            let result = epoch_day_array(name, &array, kernel)?;
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        }
+        ColumnarValue::Array(array) => {
+            Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+        }
+    }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    array: &ArrayRef,
+    kernel: F,
+) -> Result<ArrayRef> {
+    // A dictionary-encoded date column: map the dictionary values, rewrap the
+    // keys, then unpack to a plain Int32 array, so the work is proportional to
+    // the cardinality rather than the row count.
+    if let DataType::Dictionary(_, _) = array.data_type() {
+        let dict = array.as_any_dictionary();
+        let Some(values) = dict.values().as_primitive_opt::<Date32Type>() else 
{
+            return exec_err!("`{name}` expects a date, got {}", 
array.data_type());
+        };
+        let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef;
+        return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?);
+    }
+    let Some(dates) = array.as_primitive_opt::<Date32Type>() else {

Review Comment:
   Good catch, thanks. Fixed in be65c39: `epoch_day_array` now casts a `Date64` 
input to `Date32` first, including `Date64` dictionary values, so `dayofweek` 
and `weekday` handle every type their signatures accept. Both `.slt` files 
cover a scalar, a column with NULL and a date before the epoch, and 
`dayofweek.slt` also covers a dictionary. Both files fail without the fix.



##########
crates/sqllogictest/test_files/datetime/dayofweek.slt:
##########
@@ -8,7 +8,164 @@
 # For more information, please see:
 #   https://github.com/apache/datafusion/issues/15914
 
+# Spark `dayofweek` is 1-indexed with Sunday = 1 .. Saturday = 7.
+# Expected values verified against Spark 4.2.0.
+
 ## Original Query: SELECT dayofweek('2009-07-30');
 ## PySpark 3.5.5 Result: {'dayofweek(2009-07-30)': 5, 
'typeof(dayofweek(2009-07-30))': 'int', 'typeof(2009-07-30)': 'string'}
-#query
-#SELECT dayofweek('2009-07-30'::string);
+query I
+SELECT dayofweek('2009-07-30'::string);
+----
+5
+
+query I
+SELECT dayofweek('2009-07-30'::DATE);
+----
+5
+
+# All seven days of one week (2024-01-07 is a Sunday)
+query IIIIIII
+SELECT dayofweek('2024-01-07'::DATE), dayofweek('2024-01-08'::DATE), 
dayofweek('2024-01-09'::DATE),
+       dayofweek('2024-01-10'::DATE), dayofweek('2024-01-11'::DATE), 
dayofweek('2024-01-12'::DATE),
+       dayofweek('2024-01-13'::DATE);
+----
+1 2 3 4 5 6 7
+
+# Epoch, the day before it, and a leap day
+query III
+SELECT dayofweek('1970-01-01'::DATE), dayofweek('1969-12-31'::DATE), 
dayofweek('2024-02-29'::DATE);
+----
+5 4 5
+
+# Calendar boundaries. Spark uses the proleptic Gregorian calendar, so
+# 1582-10-04 (the last Julian day before the switch) is a Monday.
+query IIII
+SELECT dayofweek('0001-01-01'::DATE), dayofweek('9999-12-31'::DATE),
+       dayofweek('1582-10-04'::DATE), dayofweek('1582-10-15'::DATE);
+----
+2 6 2 6
+
+# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11
+query II
+SELECT dayofweek(date_from_unix_date(-2147483648)), 
dayofweek(date_from_unix_date(2147483647));
+----
+3 6
+
+# NULL
+query I
+SELECT dayofweek(NULL::DATE);
+----
+NULL
+
+# Untyped NULL gives a NULL INT
+query IT
+SELECT dayofweek(NULL), arrow_typeof(dayofweek(NULL));
+----
+NULL Int32
+
+# Result type
+query T
+SELECT arrow_typeof(dayofweek('2024-01-01'::DATE));
+----
+Int32
+
+# Column input, including NULL
+query I
+SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::DATE), (NULL::DATE), 
('2024-01-07'::DATE)) AS t(d);
+----
+2
+NULL
+1
+
+# Strings are implicitly cast to dates
+query I
+SELECT dayofweek('2024-05-20 23:30:00'::STRING);
+----
+2
+
+query I
+SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::STRING), (NULL::STRING), 
('2024-01-07'::STRING)) AS t(d);
+----
+2
+NULL
+1
+
+# Spark returns NULL for a malformed string when ANSI mode is off. DataFusion's

Review Comment:
   Added in be65c39 as a commented-out query with Spark's results (5 and 4, 
which I confirmed on PySpark 4.2.0 in both modes), citing #88. The 
`#[user_doc]` now says strings Spark reads as partial dates are an error as 
well.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to