This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 78ef473 [SPARK-60986][CONNECT][PYTHON] Take datetime.timedelta values
in lit() and createDataFrame as PySpark does
78ef473 is described below
commit 78ef473f5a1e2fb13a1e12f4c65994b227c788b4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 14:27:58 2026 +0900
[SPARK-60986][CONNECT][PYTHON] Take datetime.timedelta values in lit() and
createDataFrame as PySpark does
### What changes were proposed in this pull request?
[SPARK-60986](https://issues.apache.org/jira/browse/SPARK-60986): a
`datetime.timedelta` is now a value of `DayTimeIntervalType` (`INTERVAL DAY TO
SECOND`), as in PySpark 4.3.0. Its value is `((days * 86400) + seconds) *
1_000_000 + microseconds`, as `DayTimeIntervalType.toInternal` gives it.
- **`lit`:** `lit` of a `timedelta` is such a literal.
- **`createDataFrame`:** it takes a `timedelta` in a row and infers
`interval day to second`.
- **Conversion:** a day-time interval column converts its values as PyArrow
converts them to `duration[us]`, which PySpark relies on:
- an int, or a float truncated, is taken as microseconds;
- other values raise PyArrow's errors, such as `ArrowTypeError: Expected
integer, got bool`.
- **Rust:** `SparkSession::create_dataframe` writes such a column as an
Arrow duration of microseconds, as Spark's Arrow conversion does.
### Why are the changes needed?
- `F.lit(datetime.timedelta(seconds=1))` raised `UNSUPPORTED_DATA_TYPE`.
- `createDataFrame([(datetime.timedelta(seconds=1),)])` raised `TypeError:
unsupported value type for createDataFrame: timedelta`.
- The Rust `create_dataframe` had no Arrow conversion for the type.
### Does this PR introduce _any_ user-facing change?
Yes, `timedelta` values work as in PySpark 4.3.0.
### How was this patch tested?
- Against a local Spark 4.3.0 server, these give the same types, values and
errors as official PySpark 4.3.0:
- `lit` and `createDataFrame` of positive and negative `timedelta` values;
- 14 values in an explicit `interval day to second` column: timedeltas,
ints, floats, NaN, infinity, an out-of-range int, a bool, NumPy scalars, a
string, bytes and `None`.
- A new unit test checks the Arrow encoding of the column.
- A new offline test checks the conversions and errors, and a new live test
checks the types and values. The live test also passes with official PySpark
4.3.0, and fails without this change.
- The lib tests and every integration test, e2e included, against the local
server (646 passed), the build of all tests with `wasm-udf`, the offline Python
suites (648 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (64 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
The literal's repr, which PySpark renders as the ISO duration
`P1DT0H0M2.000003S`, is left to
[SPARK-60948](https://issues.apache.org/jira/browse/SPARK-60948).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #189 from HyukjinKwon/SPARK-60986-timedelta.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/functions.rs | 5 +++
crates/pyspark-rs/src/infer.rs | 77 ++++++++++++++++++++++++++++++++++++-
crates/pyspark-rs/src/session.rs | 9 +++++
crates/spark-connect/src/session.rs | 27 +++++++++++++
python/tests/test_dropin_e2e.py | 23 +++++++++++
python/tests/test_dropin_offline.py | 23 +++++++++++
6 files changed, 163 insertions(+), 1 deletion(-)
diff --git a/crates/pyspark-rs/src/functions.rs
b/crates/pyspark-rs/src/functions.rs
index 0e95069..d02da81 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -277,6 +277,11 @@ fn scalar_literal(obj: &Bound<'_, PyAny>) ->
PyResult<Option<LiteralExpression>>
let micros = crate::session::datetime_to_micros(obj)?;
return Ok(Some(LiteralExpression::Timestamp(micros)));
}
+ // datetime.timedelta -> DayTimeIntervalType, as PySpark's literal of
one.
+ if obj.is_instance(&dt_mod.getattr("timedelta")?)? {
+ let micros = crate::infer::timedelta_micros(obj)?;
+ return Ok(Some(LiteralExpression::DayTimeInterval(micros)));
+ }
let date_cls = dt_mod.getattr("date")?;
if obj.is_instance(&date_cls)? {
let ordinal: i64 = obj.call_method0("toordinal")?.extract()?;
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index c820800..0119d54 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -7,7 +7,7 @@
use std::cell::OnceCell;
use std::collections::{HashMap, HashSet};
-use pyo3::exceptions::{PyAssertionError, PyTypeError, PyValueError};
+use pyo3::exceptions::{PyAssertionError, PyOverflowError, PyTypeError,
PyValueError};
use pyo3::intern;
use pyo3::prelude::*;
use pyo3::sync::PyOnceLock;
@@ -1042,6 +1042,7 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
_ => Value::Double(value),
})
}
+ DataType::DayTimeInterval { .. } =>
Ok(Value::DayTimeInterval(duration_micros(obj)?)),
DataType::Date => {
// PyArrow takes a date, or the date of a datetime.
if !obj.is_instance(DATE.import(py, "datetime",
"date")?.as_any())? {
@@ -1054,6 +1055,80 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
}
}
+/// A value of a day-time interval column, as PyArrow converts one to
`duration[us]`: a
+/// timedelta to its microseconds, as `DayTimeIntervalType.toInternal`, and an
int, or a
+/// float truncated, as microseconds. Another value raises PyArrow's error.
+fn duration_micros(obj: &Bound<'_, PyAny>) -> PyResult<i64> {
+ let py = obj.py();
+ if obj.is_instance(TIMEDELTA.import(py, "datetime",
"timedelta")?.as_any())? {
+ return timedelta_micros(obj);
+ }
+ let pyarrow_error = |class: &str, message: String| -> PyResult<PyErr> {
+ let error = py
+ .import(intern!(py, "pyarrow"))?
+ .getattr(class)?
+ .call1((message,))?;
+ Ok(PyErr::from_value(error))
+ };
+ if obj.is_instance_of::<PyBool>() {
+ return Err(pyarrow_error(
+ "ArrowTypeError",
+ "Expected integer, got bool".into(),
+ )?);
+ }
+ if numpy_scalar(obj)?.is_some() {
+ let arrow_type = py.import(intern!(py, "pyarrow"))?.call_method1(
+ intern!(py, "from_numpy_dtype"),
+ (obj.getattr(intern!(py, "dtype"))?,),
+ )?;
+ return Err(pyarrow_error(
+ "ArrowNotImplementedError",
+ format!("Expected np.timedelta64 but got: {}", arrow_type.str()?),
+ )?);
+ }
+ let too_large = || PyOverflowError::new_err("Python int too large to
convert to C long");
+ if obj.is_instance_of::<PyInt>() {
+ return obj.extract::<i64>().map_err(|_| too_large());
+ }
+ if let Ok(value) = obj.cast::<PyFloat>() {
+ let value = value.value();
+ if value.is_nan() {
+ return Err(PyValueError::new_err("cannot convert float NaN to
integer"));
+ }
+ if value.is_infinite() {
+ return Err(PyOverflowError::new_err(
+ "cannot convert float infinity to integer",
+ ));
+ }
+ let truncated = value.trunc();
+ if
!(-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&truncated)
{
+ return Err(too_large());
+ }
+ return Ok(truncated as i64);
+ }
+ Err(pyarrow_error(
+ "ArrowTypeError",
+ format!(
+ "object of type {} cannot be converted to int",
+ obj.get_type().repr()?
+ ),
+ )?)
+}
+
+/// The microseconds of a `datetime.timedelta`, as
`DayTimeIntervalType.toInternal`:/// The microseconds of a
`datetime.timedelta`, as `DayTimeIntervalType.toInternal`:
+/// `((days * 86400) + seconds) * 1_000_000 + microseconds`.
+pub(crate) fn timedelta_micros(obj: &Bound<'_, PyAny>) -> PyResult<i64> {
+ let py = obj.py();
+ let days: i64 = obj.getattr(intern!(py, "days"))?.extract()?;
+ let seconds: i64 = obj.getattr(intern!(py, "seconds"))?.extract()?;
+ let micros: i64 = obj.getattr(intern!(py, "microseconds"))?.extract()?;
+ days.checked_mul(86_400)
+ .and_then(|s| s.checked_add(seconds))
+ .and_then(|s| s.checked_mul(1_000_000))
+ .and_then(|us| us.checked_add(micros))
+ .ok_or_else(|| PyOverflowError::new_err("Python int too large to
convert to C long"))
+}
+
/// An integral value, as PyArrow converts one: an int in range, or a float
truncated.
fn convert_integral(obj: &Bound<'_, PyAny>, dt: &DataType) -> PyResult<Value> {
let (bits, arrow_type) = match dt {
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 433926d..76895af 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1532,6 +1532,15 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) ->
PyResult<Value> {
}
}
+ // datetime.timedelta -> a day-time interval, as
DayTimeIntervalType.toInternal.
+ if let Some(dt_mod) = &datetime_mod {
+ if let Ok(timedelta_cls) = dt_mod.getattr("timedelta") {
+ if obj.is_instance(&timedelta_cls)? {
+ return
Ok(Value::DayTimeInterval(crate::infer::timedelta_micros(obj)?));
+ }
+ }
+ }
+
// Check for datetime.date (after datetime.datetime)
if let Some(dt_mod) = &datetime_mod {
if let Ok(date_cls) = dt_mod.getattr("date") {
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index ab57f64..6251451 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -874,6 +874,8 @@ fn datatype_to_arrow(dt: &DataType) ->
Result<arrow::datatypes::DataType> {
}
DataType::TimestampNtzNanos { .. } =>
A::Timestamp(TimeUnit::Nanosecond, None),
DataType::Time { .. } => A::Time64(TimeUnit::Microsecond),
+ // As Spark's Arrow conversion of DayTimeIntervalType.
+ DataType::DayTimeInterval { .. } => A::Duration(TimeUnit::Microsecond),
DataType::Decimal { precision, scale } => A::Decimal128(*precision as
u8, *scale as i8),
DataType::Array {
element_type,
@@ -1013,6 +1015,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
Ok(Arc::new(BinaryArray::from(vals?)))
}
DataType::Date => prim!(Date32Array, Value::Date),
+ DataType::DayTimeInterval { .. } => prim!(DurationMicrosecondArray,
Value::DayTimeInterval),
DataType::Timestamp | DataType::TimestampNtz => {
let vals: Result<Vec<Option<i64>>> = values
.iter()
@@ -1606,6 +1609,30 @@ mod tests {
batch
}
+ #[test]
+ fn input_ipc_writes_day_time_intervals_as_microsecond_durations() {
+ // As Spark's Arrow conversion of DayTimeIntervalType.
+ let interval = DataType::DayTimeInterval {
+ start_field: 0,
+ end_field: 3,
+ };
+ let rows = vec![
+ row(vec![Value::DayTimeInterval(86_402_000_003)]),
+ row(vec![Value::Null]),
+ row(vec![Value::DayTimeInterval(-999_995)]),
+ ];
+ let batch = read_batch(&rows, &schema(vec![interval]));
+ let column = batch
+ .column(0)
+ .as_any()
+ .downcast_ref::<arrow::array::DurationMicrosecondArray>()
+ .unwrap();
+ assert_eq!(
+ column.iter().collect::<Vec<_>>(),
+ vec![Some(86_402_000_003), None, Some(-999_995)]
+ );
+ }
+
#[test]
fn input_ipc_preserves_scalar_coercion_and_nulls() {
let schema = schema(vec![
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 30971b9..bc0bc9b 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -429,6 +429,29 @@ def
test_collect_reads_a_result_across_stream_ends(session):
assert session.range(n).toArrow().num_rows == n
+def test_timedelta_is_a_day_time_interval_as_pyspark(session):
+ import datetime
+
+ import pyarrow as pa
+
+ values = [datetime.timedelta(days=1, seconds=2, microseconds=3),
+ datetime.timedelta(seconds=-1, microseconds=5)]
+ df = session.range(1).select(*[F.lit(v).alias(f"t{i}") for i, v in
enumerate(values)])
+ assert df.dtypes == [("t0", "interval day to second"), ("t1", "interval
day to second")]
+ assert tuple(df.first()) == tuple(values)
+ df = session.createDataFrame([(v,) for v in values] + [(None,)], ["t"])
+ assert df.dtypes == [("t", "interval day to second")]
+ assert [r.t for r in df.collect()] == values + [None]
+ # As PyArrow's conversion to duration[us]: an int, or a float truncated,
is microseconds.
+ df = session.createDataFrame([(5,), (1.5,)], "d interval day to second")
+ assert [r.d for r in df.collect()] == [
+ datetime.timedelta(microseconds=5), datetime.timedelta(microseconds=1)]
+ with pytest.raises(pa.ArrowTypeError, match="Expected integer, got bool"):
+ session.createDataFrame([(True,)], "d interval day to second")
+ with pytest.raises(pa.ArrowTypeError, match="object of type <class 'str'>
cannot be converted to int"):
+ session.createDataFrame([("x",)], "d interval day to second")
+
+
def test_server_errors_carry_their_condition_and_details(session):
from pyspark.errors import AnalysisException, ArithmeticException,
ParseException
from pyspark.errors.exceptions.connect import SparkConnectGrpcException,
SQLQueryContext
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 7393ff4..d7d7296 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3625,6 +3625,29 @@ def
test_create_dataframe_types_pandas_columns_as_arrow():
_local_create_dataframe(pd.DataFrame({"m": [{None: 1}]}))
+def test_timedelta_values_convert_as_pyspark():
+ import datetime
+ import re
+
+ pa = pytest.importorskip("pyarrow")
+ from pyspark.sql import functions as F
+
+ # A day-time interval, rather than UNSUPPORTED_DATA_TYPE (the live suite
checks values).
+ one = datetime.timedelta(days=1, seconds=2, microseconds=3)
+ F.lit(one)
+ _local_create_dataframe([(one,), (None,)])
+ _local_create_dataframe([(5,), (1.5,)], "d interval day to second")
+ # Other values raise as PyArrow's conversion to duration[us] does.
+ for value, error, message in (
+ (True, pa.ArrowTypeError, "Expected integer, got bool"),
+ ("x", pa.ArrowTypeError, "object of type <class 'str'> cannot be
converted to int"),
+ (float("nan"), ValueError, "cannot convert float NaN to integer"),
+ (2**63, OverflowError, "Python int too large to convert to C long"),
+ ):
+ with pytest.raises(error, match="^" + re.escape(message) + "$"):
+ _local_create_dataframe([(value,)], "d interval day to second")
+
+
def _local_create_dataframe(*args):
# The local relation and its toDF are built without a server.
return _offline_session().createDataFrame(*args)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]