mbutrovich commented on code in PR #327:
URL: https://github.com/apache/datafusion-spark/pull/327#discussion_r4602515069


##########
crates/sqllogictest/test_files/datetime/dayofweek.slt:
##########
@@ -8,7 +8,164 @@
 # For more information, please see:
 #   https://github.com/apache/datafusion/issues/15914
 
+# Spark `dayofweek` is 1-indexed with Sunday = 1 .. Saturday = 7.
+# Expected values verified against Spark 4.2.0.
+
 ## Original Query: SELECT dayofweek('2009-07-30');
 ## PySpark 3.5.5 Result: {'dayofweek(2009-07-30)': 5, 
'typeof(dayofweek(2009-07-30))': 'int', 'typeof(2009-07-30)': 'string'}
-#query
-#SELECT dayofweek('2009-07-30'::string);
+query I
+SELECT dayofweek('2009-07-30'::string);
+----
+5
+
+query I
+SELECT dayofweek('2009-07-30'::DATE);
+----
+5
+
+# All seven days of one week (2024-01-07 is a Sunday)
+query IIIIIII
+SELECT dayofweek('2024-01-07'::DATE), dayofweek('2024-01-08'::DATE), 
dayofweek('2024-01-09'::DATE),
+       dayofweek('2024-01-10'::DATE), dayofweek('2024-01-11'::DATE), 
dayofweek('2024-01-12'::DATE),
+       dayofweek('2024-01-13'::DATE);
+----
+1 2 3 4 5 6 7
+
+# Epoch, the day before it, and a leap day
+query III
+SELECT dayofweek('1970-01-01'::DATE), dayofweek('1969-12-31'::DATE), 
dayofweek('2024-02-29'::DATE);
+----
+5 4 5
+
+# Calendar boundaries. Spark uses the proleptic Gregorian calendar, so
+# 1582-10-04 (the last Julian day before the switch) is a Monday.
+query IIII
+SELECT dayofweek('0001-01-01'::DATE), dayofweek('9999-12-31'::DATE),
+       dayofweek('1582-10-04'::DATE), dayofweek('1582-10-15'::DATE);
+----
+2 6 2 6
+
+# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11
+query II
+SELECT dayofweek(date_from_unix_date(-2147483648)), 
dayofweek(date_from_unix_date(2147483647));
+----
+3 6
+
+# NULL
+query I
+SELECT dayofweek(NULL::DATE);
+----
+NULL
+
+# Untyped NULL gives a NULL INT
+query IT
+SELECT dayofweek(NULL), arrow_typeof(dayofweek(NULL));
+----
+NULL Int32
+
+# Result type
+query T
+SELECT arrow_typeof(dayofweek('2024-01-01'::DATE));
+----
+Int32
+
+# Column input, including NULL
+query I
+SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::DATE), (NULL::DATE), 
('2024-01-07'::DATE)) AS t(d);
+----
+2
+NULL
+1
+
+# Strings are implicitly cast to dates
+query I
+SELECT dayofweek('2024-05-20 23:30:00'::STRING);
+----
+2
+
+query I
+SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::STRING), (NULL::STRING), 
('2024-01-07'::STRING)) AS t(d);
+----
+2
+NULL
+1
+
+# Spark returns NULL for a malformed string when ANSI mode is off. DataFusion's

Review Comment:
   Spark also accepts partial dates as strings, in both ANSI modes. On Spark 
4.2.0, `SELECT dayofweek('2009')` returns 5 and `SELECT dayofweek('2009-07')` 
returns 4. DataFusion's cast rejects both with ANSI on or off (`Cannot cast 
string '2009' to value of Date32 type`).
   
   This comes from `date_signature()` on `main` 
([`date_part_name.rs`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/crates/spark/src/function/datetime/date_part_name.rs#L19-L35)),
 so `dayname` and `monthname` behave the same way. The "String to date" item in 
#88 covers the fix. Could we add these two as commented-out queries with 
Spark's results next to this one? Could the `#[user_doc]` description also say 
that strings Spark reads as partial dates raise an error, not only malformed 
ones?



##########
crates/spark/src/function/datetime/dayofweek.rs:
##########
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array};
+use arrow::compute::cast;
+use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, internal_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, 
ScalarUDFImpl,
+    Signature,
+};
+use datafusion_macros::user_doc;
+
+use super::date_part_name::date_signature;
+
+/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`,
+/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, 
so
+/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` 
first
+/// keeps `i32::MAX` from overflowing.
+#[inline]
+pub(super) fn day_of_week(days: i32) -> i32 {
+    ((i64::from(days) + 4).rem_euclid(7) + 1) as i32
+}
+
+/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6.
+///
+/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`.
+#[inline]
+pub(super) fn week_day(days: i32) -> i32 {
+    (i64::from(days) + 3).rem_euclid(7) as i32
+}
+
+/// Applies `kernel` to every valid date.
+///
+/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid
+/// slots but costs more per element, so switch on null density.
+#[inline]
+fn map_dates<F: Fn(i32) -> i32 + Copy>(dates: &Date32Array, kernel: F) -> 
Int32Array {
+    if dates.null_count() * 2 <= dates.len() {
+        dates.unary::<_, Int32Type>(kernel)
+    } else {
+        dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d)))
+    }
+}
+
+/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly
+/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL.
+///
+/// The value is computed straight from the epoch day rather than through a
+/// calendar date, so it is defined for the whole `i32` domain as in Spark.
+pub(super) fn epoch_day_field<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    arg: ColumnarValue,
+    kernel: F,
+) -> Result<ColumnarValue> {
+    match arg {
+        ColumnarValue::Scalar(ScalarValue::Date32(days)) => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel))))
+        }
+        ColumnarValue::Scalar(scalar) if scalar.is_null() => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Int32(None)))
+        }
+        ColumnarValue::Scalar(scalar) => {
+            let array = scalar.to_array_of_size(1)?;
+            let result = epoch_day_array(name, &array, kernel)?;
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        }
+        ColumnarValue::Array(array) => {
+            Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?))
+        }
+    }
+}
+
+fn epoch_day_array<F: Fn(i32) -> i32 + Copy>(
+    name: &str,
+    array: &ArrayRef,
+    kernel: F,
+) -> Result<ArrayRef> {
+    // A dictionary-encoded date column: map the dictionary values, rewrap the
+    // keys, then unpack to a plain Int32 array, so the work is proportional to
+    // the cardinality rather than the row count.
+    if let DataType::Dictionary(_, _) = array.data_type() {
+        let dict = array.as_any_dictionary();
+        let Some(values) = dict.values().as_primitive_opt::<Date32Type>() else 
{
+            return exec_err!("`{name}` expects a date, got {}", 
array.data_type());
+        };
+        let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef;
+        return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?);
+    }
+    let Some(dates) = array.as_primitive_opt::<Date32Type>() else {

Review Comment:
   `date_signature()` matches the logical date type, so it also admits 
`Date64`, but this only handles `Date32`. `SELECT 
dayofweek(arrow_cast('2024-01-07', 'Date64'))` plans and then fails with 
``Execution error: `dayofweek` expects a date, got Date64``.
   
   `weekday` has the same problem after this PR. On `main` it goes through 
arrow's `date_part`, which handles `Date64`, and `SELECT 
weekday(arrow_cast('2024-01-07', 'Date64'))` returns 6. With this PR it fails 
with the same error.
   
   Could we cast a `Date64` input to `Date32` at the top of `epoch_day_array`? 
Then both functions handle every type their signatures accept.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to