This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 78ef473  [SPARK-60986][CONNECT][PYTHON] Take datetime.timedelta values 
in lit() and createDataFrame as PySpark does
78ef473 is described below

commit 78ef473f5a1e2fb13a1e12f4c65994b227c788b4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 14:27:58 2026 +0900

    [SPARK-60986][CONNECT][PYTHON] Take datetime.timedelta values in lit() and 
createDataFrame as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60986](https://issues.apache.org/jira/browse/SPARK-60986): a 
`datetime.timedelta` is now a value of `DayTimeIntervalType` (`INTERVAL DAY TO 
SECOND`), as in PySpark 4.3.0. Its value is `((days * 86400) + seconds) * 
1_000_000 + microseconds`, as `DayTimeIntervalType.toInternal` gives it.
    
    - **`lit`:** `lit` of a `timedelta` is such a literal.
    - **`createDataFrame`:** it takes a `timedelta` in a row and infers 
`interval day to second`.
    - **Conversion:** a day-time interval column converts its values as PyArrow 
converts them to `duration[us]`, which PySpark relies on:
      - an int, or a float truncated, is taken as microseconds;
      - other values raise PyArrow's errors, such as `ArrowTypeError: Expected 
integer, got bool`.
    - **Rust:** `SparkSession::create_dataframe` writes such a column as an 
Arrow duration of microseconds, as Spark's Arrow conversion does.
    
    ### Why are the changes needed?
    
    - `F.lit(datetime.timedelta(seconds=1))` raised `UNSUPPORTED_DATA_TYPE`.
    - `createDataFrame([(datetime.timedelta(seconds=1),)])` raised `TypeError: 
unsupported value type for createDataFrame: timedelta`.
    - The Rust `create_dataframe` had no Arrow conversion for the type.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `timedelta` values work as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, these give the same types, values and 
errors as official PySpark 4.3.0:
      - `lit` and `createDataFrame` of positive and negative `timedelta` values;
      - 14 values in an explicit `interval day to second` column: timedeltas, 
ints, floats, NaN, infinity, an out-of-range int, a bool, NumPy scalars, a 
string, bytes and `None`.
    - A new unit test checks the Arrow encoding of the column.
    - A new offline test checks the conversions and errors, and a new live test 
checks the types and values. The live test also passes with official PySpark 
4.3.0, and fails without this change.
    - The lib tests and every integration test, e2e included, against the local 
server (646 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (648 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (64 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    The literal's repr, which PySpark renders as the ISO duration 
`P1DT0H0M2.000003S`, is left to 
[SPARK-60948](https://issues.apache.org/jira/browse/SPARK-60948).
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #189 from HyukjinKwon/SPARK-60986-timedelta.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs  |  5 +++
 crates/pyspark-rs/src/infer.rs      | 77 ++++++++++++++++++++++++++++++++++++-
 crates/pyspark-rs/src/session.rs    |  9 +++++
 crates/spark-connect/src/session.rs | 27 +++++++++++++
 python/tests/test_dropin_e2e.py     | 23 +++++++++++
 python/tests/test_dropin_offline.py | 23 +++++++++++
 6 files changed, 163 insertions(+), 1 deletion(-)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index 0e95069..d02da81 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -277,6 +277,11 @@ fn scalar_literal(obj: &Bound<'_, PyAny>) -> 
PyResult<Option<LiteralExpression>>
             let micros = crate::session::datetime_to_micros(obj)?;
             return Ok(Some(LiteralExpression::Timestamp(micros)));
         }
+        // datetime.timedelta -> DayTimeIntervalType, as PySpark's literal of 
one.
+        if obj.is_instance(&dt_mod.getattr("timedelta")?)? {
+            let micros = crate::infer::timedelta_micros(obj)?;
+            return Ok(Some(LiteralExpression::DayTimeInterval(micros)));
+        }
         let date_cls = dt_mod.getattr("date")?;
         if obj.is_instance(&date_cls)? {
             let ordinal: i64 = obj.call_method0("toordinal")?.extract()?;
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index c820800..0119d54 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -7,7 +7,7 @@
 use std::cell::OnceCell;
 use std::collections::{HashMap, HashSet};
 
-use pyo3::exceptions::{PyAssertionError, PyTypeError, PyValueError};
+use pyo3::exceptions::{PyAssertionError, PyOverflowError, PyTypeError, 
PyValueError};
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::sync::PyOnceLock;
@@ -1042,6 +1042,7 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt: 
&DataType, nullable: bool) ->
                 _ => Value::Double(value),
             })
         }
+        DataType::DayTimeInterval { .. } => 
Ok(Value::DayTimeInterval(duration_micros(obj)?)),
         DataType::Date => {
             // PyArrow takes a date, or the date of a datetime.
             if !obj.is_instance(DATE.import(py, "datetime", 
"date")?.as_any())? {
@@ -1054,6 +1055,80 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt: 
&DataType, nullable: bool) ->
     }
 }
 
+/// A value of a day-time interval column, as PyArrow converts one to 
`duration[us]`: a
+/// timedelta to its microseconds, as `DayTimeIntervalType.toInternal`, and an 
int, or a
+/// float truncated, as microseconds. Another value raises PyArrow's error.
+fn duration_micros(obj: &Bound<'_, PyAny>) -> PyResult<i64> {
+    let py = obj.py();
+    if obj.is_instance(TIMEDELTA.import(py, "datetime", 
"timedelta")?.as_any())? {
+        return timedelta_micros(obj);
+    }
+    let pyarrow_error = |class: &str, message: String| -> PyResult<PyErr> {
+        let error = py
+            .import(intern!(py, "pyarrow"))?
+            .getattr(class)?
+            .call1((message,))?;
+        Ok(PyErr::from_value(error))
+    };
+    if obj.is_instance_of::<PyBool>() {
+        return Err(pyarrow_error(
+            "ArrowTypeError",
+            "Expected integer, got bool".into(),
+        )?);
+    }
+    if numpy_scalar(obj)?.is_some() {
+        let arrow_type = py.import(intern!(py, "pyarrow"))?.call_method1(
+            intern!(py, "from_numpy_dtype"),
+            (obj.getattr(intern!(py, "dtype"))?,),
+        )?;
+        return Err(pyarrow_error(
+            "ArrowNotImplementedError",
+            format!("Expected np.timedelta64 but got: {}", arrow_type.str()?),
+        )?);
+    }
+    let too_large = || PyOverflowError::new_err("Python int too large to 
convert to C long");
+    if obj.is_instance_of::<PyInt>() {
+        return obj.extract::<i64>().map_err(|_| too_large());
+    }
+    if let Ok(value) = obj.cast::<PyFloat>() {
+        let value = value.value();
+        if value.is_nan() {
+            return Err(PyValueError::new_err("cannot convert float NaN to 
integer"));
+        }
+        if value.is_infinite() {
+            return Err(PyOverflowError::new_err(
+                "cannot convert float infinity to integer",
+            ));
+        }
+        let truncated = value.trunc();
+        if 
!(-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&truncated)
 {
+            return Err(too_large());
+        }
+        return Ok(truncated as i64);
+    }
+    Err(pyarrow_error(
+        "ArrowTypeError",
+        format!(
+            "object of type {} cannot be converted to int",
+            obj.get_type().repr()?
+        ),
+    )?)
+}
+
+/// The microseconds of a `datetime.timedelta`, as 
`DayTimeIntervalType.toInternal`:/// The microseconds of a 
`datetime.timedelta`, as `DayTimeIntervalType.toInternal`:
+/// `((days * 86400) + seconds) * 1_000_000 + microseconds`.
+pub(crate) fn timedelta_micros(obj: &Bound<'_, PyAny>) -> PyResult<i64> {
+    let py = obj.py();
+    let days: i64 = obj.getattr(intern!(py, "days"))?.extract()?;
+    let seconds: i64 = obj.getattr(intern!(py, "seconds"))?.extract()?;
+    let micros: i64 = obj.getattr(intern!(py, "microseconds"))?.extract()?;
+    days.checked_mul(86_400)
+        .and_then(|s| s.checked_add(seconds))
+        .and_then(|s| s.checked_mul(1_000_000))
+        .and_then(|us| us.checked_add(micros))
+        .ok_or_else(|| PyOverflowError::new_err("Python int too large to 
convert to C long"))
+}
+
 /// An integral value, as PyArrow converts one: an int in range, or a float 
truncated.
 fn convert_integral(obj: &Bound<'_, PyAny>, dt: &DataType) -> PyResult<Value> {
     let (bits, arrow_type) = match dt {
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 433926d..76895af 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1532,6 +1532,15 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) -> 
PyResult<Value> {
         }
     }
 
+    // datetime.timedelta -> a day-time interval, as 
DayTimeIntervalType.toInternal.
+    if let Some(dt_mod) = &datetime_mod {
+        if let Ok(timedelta_cls) = dt_mod.getattr("timedelta") {
+            if obj.is_instance(&timedelta_cls)? {
+                return 
Ok(Value::DayTimeInterval(crate::infer::timedelta_micros(obj)?));
+            }
+        }
+    }
+
     // Check for datetime.date (after datetime.datetime)
     if let Some(dt_mod) = &datetime_mod {
         if let Ok(date_cls) = dt_mod.getattr("date") {
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index ab57f64..6251451 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -874,6 +874,8 @@ fn datatype_to_arrow(dt: &DataType) -> 
Result<arrow::datatypes::DataType> {
         }
         DataType::TimestampNtzNanos { .. } => 
A::Timestamp(TimeUnit::Nanosecond, None),
         DataType::Time { .. } => A::Time64(TimeUnit::Microsecond),
+        // As Spark's Arrow conversion of DayTimeIntervalType.
+        DataType::DayTimeInterval { .. } => A::Duration(TimeUnit::Microsecond),
         DataType::Decimal { precision, scale } => A::Decimal128(*precision as 
u8, *scale as i8),
         DataType::Array {
             element_type,
@@ -1013,6 +1015,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) -> 
Result<Arc<dyn arrow::arr
             Ok(Arc::new(BinaryArray::from(vals?)))
         }
         DataType::Date => prim!(Date32Array, Value::Date),
+        DataType::DayTimeInterval { .. } => prim!(DurationMicrosecondArray, 
Value::DayTimeInterval),
         DataType::Timestamp | DataType::TimestampNtz => {
             let vals: Result<Vec<Option<i64>>> = values
                 .iter()
@@ -1606,6 +1609,30 @@ mod tests {
         batch
     }
 
+    #[test]
+    fn input_ipc_writes_day_time_intervals_as_microsecond_durations() {
+        // As Spark's Arrow conversion of DayTimeIntervalType.
+        let interval = DataType::DayTimeInterval {
+            start_field: 0,
+            end_field: 3,
+        };
+        let rows = vec![
+            row(vec![Value::DayTimeInterval(86_402_000_003)]),
+            row(vec![Value::Null]),
+            row(vec![Value::DayTimeInterval(-999_995)]),
+        ];
+        let batch = read_batch(&rows, &schema(vec![interval]));
+        let column = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow::array::DurationMicrosecondArray>()
+            .unwrap();
+        assert_eq!(
+            column.iter().collect::<Vec<_>>(),
+            vec![Some(86_402_000_003), None, Some(-999_995)]
+        );
+    }
+
     #[test]
     fn input_ipc_preserves_scalar_coercion_and_nulls() {
         let schema = schema(vec![
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 30971b9..bc0bc9b 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -429,6 +429,29 @@ def 
test_collect_reads_a_result_across_stream_ends(session):
     assert session.range(n).toArrow().num_rows == n
 
 
+def test_timedelta_is_a_day_time_interval_as_pyspark(session):
+    import datetime
+
+    import pyarrow as pa
+
+    values = [datetime.timedelta(days=1, seconds=2, microseconds=3),
+              datetime.timedelta(seconds=-1, microseconds=5)]
+    df = session.range(1).select(*[F.lit(v).alias(f"t{i}") for i, v in 
enumerate(values)])
+    assert df.dtypes == [("t0", "interval day to second"), ("t1", "interval 
day to second")]
+    assert tuple(df.first()) == tuple(values)
+    df = session.createDataFrame([(v,) for v in values] + [(None,)], ["t"])
+    assert df.dtypes == [("t", "interval day to second")]
+    assert [r.t for r in df.collect()] == values + [None]
+    # As PyArrow's conversion to duration[us]: an int, or a float truncated, 
is microseconds.
+    df = session.createDataFrame([(5,), (1.5,)], "d interval day to second")
+    assert [r.d for r in df.collect()] == [
+        datetime.timedelta(microseconds=5), datetime.timedelta(microseconds=1)]
+    with pytest.raises(pa.ArrowTypeError, match="Expected integer, got bool"):
+        session.createDataFrame([(True,)], "d interval day to second")
+    with pytest.raises(pa.ArrowTypeError, match="object of type <class 'str'> 
cannot be converted to int"):
+        session.createDataFrame([("x",)], "d interval day to second")
+
+
 def test_server_errors_carry_their_condition_and_details(session):
     from pyspark.errors import AnalysisException, ArithmeticException, 
ParseException
     from pyspark.errors.exceptions.connect import SparkConnectGrpcException, 
SQLQueryContext
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 7393ff4..d7d7296 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3625,6 +3625,29 @@ def 
test_create_dataframe_types_pandas_columns_as_arrow():
         _local_create_dataframe(pd.DataFrame({"m": [{None: 1}]}))
 
 
+def test_timedelta_values_convert_as_pyspark():
+    import datetime
+    import re
+
+    pa = pytest.importorskip("pyarrow")
+    from pyspark.sql import functions as F
+
+    # A day-time interval, rather than UNSUPPORTED_DATA_TYPE (the live suite 
checks values).
+    one = datetime.timedelta(days=1, seconds=2, microseconds=3)
+    F.lit(one)
+    _local_create_dataframe([(one,), (None,)])
+    _local_create_dataframe([(5,), (1.5,)], "d interval day to second")
+    # Other values raise as PyArrow's conversion to duration[us] does.
+    for value, error, message in (
+        (True, pa.ArrowTypeError, "Expected integer, got bool"),
+        ("x", pa.ArrowTypeError, "object of type <class 'str'> cannot be 
converted to int"),
+        (float("nan"), ValueError, "cannot convert float NaN to integer"),
+        (2**63, OverflowError, "Python int too large to convert to C long"),
+    ):
+        with pytest.raises(error, match="^" + re.escape(message) + "$"):
+            _local_create_dataframe([(value,)], "d interval day to second")
+
+
 def _local_create_dataframe(*args):
     # The local relation and its toDF are built without a server.
     return _offline_session().createDataFrame(*args)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to