This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 5a3aac3 [SPARK-60659][CONNECT] Test day-time interval inputs, and
document them
5a3aac3 is described below
commit 5a3aac3603076af54e40395130a3695c881a8aa2
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 18:27:20 2026 +0900
[SPARK-60659][CONNECT] Test day-time interval inputs, and document them
### What changes were proposed in this pull request?
Tests for day-time interval inputs, and a fix of the docs that said they
were unsupported. The conversion itself landed in master with SPARK-60986
(#189), which takes `timedelta` values, and ints and floats as microseconds, in
`lit()` and `createDataFrame` as PySpark does; this branch's own conversion,
which came before master's and rejected ints and floats, is dropped in favor of
master's.
- A Rust unit test encodes `Value::DayTimeInterval` at the int64 bounds,
with nulls, in arrays, maps and structs, decodes the IPC back, and checks that
a duration field rejects an untyped integer.
- A Rust e2e test creates DataFrames of interval rows (scalars, arrays,
structs and maps, the int64 bounds and nulls) against a Spark server, collects
them and creates them again from the collected rows, and does the same for an
interval of a SQL query.
- `python/tests/interval_input_cases.py` checks overflow at the top level
and nested, before any RPC; the `CANNOT_MERGE_TYPE` and
`CANNOT_INFER_TYPE_FOR_FIELD` errors of intervals mixed with numbers; the
bounds that PySpark takes (down to -106751991 days, whose microseconds fit in
64 bits, and up to 2^63 - 1); ints as microseconds; and the exact microseconds
of pandas object columns. Its conversion tests run in the offline suite; its
live tests, against a server, run with `SPARK_REMOTE`.
- `docs/architecture.md` describes interval input instead of pointing at
this JIRA as unsupported.
### Why are the changes needed?
[SPARK-60659](https://issues.apache.org/jira/browse/SPARK-60659): the
support of day-time interval inputs had no Rust-level or bound tests, and the
docs still said it was missing.
### Does this PR introduce _any_ user-facing change?
No, tests and docs only.
### How was this patch tested?
- The live tests of `interval_input_cases.py` pass against a local Spark
4.3.0 server. Official PySpark 4.3.0 takes and rejects the same bounds.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (785 passed), the build of all tests
with `wasm-udf`, the offline Python suites (891 passed, 9 skipped), the live
drop-in and Arrow suites (81 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master and the
adaptation to master's conversion by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #116 from dbtsai/codex/spark-60659-interval-input.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/session.rs | 76 ++++++++++
crates/spark-connect/tests/e2e_dataframe.rs | 108 ++++++++++++++
docs/architecture.md | 9 +-
python/tests/interval_input_cases.py | 211 ++++++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 1 +
5 files changed, 402 insertions(+), 3 deletions(-)
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 761288f..5489e14 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -1338,6 +1338,82 @@ mod tests {
}
}
+ #[test]
+ fn interval_input_ipc_is_exact_and_recursive() {
+ let interval = DataType::DayTimeInterval {
+ start_field: 0,
+ end_field: 3,
+ };
+ let nested = DataType::Struct {
+ fields: vec![
+ struct_field("leaf", interval.clone()),
+ struct_field(
+ "items",
+ DataType::Array {
+ element_type: Box::new(interval.clone()),
+ contains_null: true,
+ },
+ ),
+ struct_field(
+ "mapping",
+ DataType::Map {
+ key_type: Box::new(text_type()),
+ value_type: Box::new(interval.clone()),
+ value_contains_null: true,
+ },
+ ),
+ ],
+ };
+ let schema = schema(vec![interval, nested]);
+ let mut rows = vec![row(vec![Value::Null, Value::Null])];
+ for micros in [
+ i64::MIN,
+ i64::MIN + 1,
+ -172_800_000_001,
+ -1,
+ 0,
+ 1,
+ i64::MAX - 1,
+ i64::MAX,
+ ] {
+ let value = Value::DayTimeInterval(micros);
+ rows.push(row(vec![
+ value.clone(),
+ Value::Struct(
+ Row::new(
+ vec!["leaf".into(), "items".into(), "mapping".into()],
+ vec![
+ value.clone(),
+ Value::List(vec![Value::Null, value.clone()]),
+ Value::Map(vec![
+ (Value::String("k".into()), value),
+ (Value::String("null".into()), Value::Null),
+ ]),
+ ],
+ )
+ .into(),
+ ),
+ ]));
+ }
+ let batch = read_batch(&rows, &schema);
+ assert_eq!(
+ batch.column(0).data_type(),
+
&arrow::datatypes::DataType::Duration(arrow::datatypes::TimeUnit::Microsecond)
+ );
+ let decoded =
+
crate::dataframe::decode_arrow_batch(&proto::execute_plan_response::ArrowBatch {
+ row_count: rows.len() as i64,
+ data: rows_to_arrow_ipc(&rows, &schema).unwrap().into(),
+ ..Default::default()
+ })
+ .unwrap();
+ for (actual, expected) in decoded.iter().zip(&rows) {
+ assert_eq!(actual.values(), expected.values());
+ }
+ // A duration schema must reject untyped integers rather than silently
treating them as micros.
+ assert!(rows_to_arrow_ipc(&[row(vec![Value::Long(1), Value::Null])],
&schema).is_err());
+ }
+
#[test]
fn small_integer_rows_encode_as_floating_ipc() {
use arrow::array::{Array, Float32Array, Float64Array, ListArray,
MapArray, StructArray};
diff --git a/crates/spark-connect/tests/e2e_dataframe.rs
b/crates/spark-connect/tests/e2e_dataframe.rs
index de2ec9b..e783a50 100644
--- a/crates/spark-connect/tests/e2e_dataframe.rs
+++ b/crates/spark-connect/tests/e2e_dataframe.rs
@@ -131,3 +131,111 @@ fn dataframe_metadata_json_semantics() {
let parts = df.random_split(vec![0.5, 0.5], Some(1));
assert_eq!(parts.len(), 2);
}
+
+#[test]
+fn day_time_interval_input_roundtrips() {
+ if !should_run() {
+ return;
+ }
+ use spark_connect::row::{Row, Value};
+ use spark_connect::types::{DataType, StructField};
+
+ fn field(name: &str, data_type: DataType) -> StructField {
+ StructField {
+ name: name.into(),
+ data_type,
+ nullable: true,
+ metadata: Default::default(),
+ }
+ }
+ let s = session();
+ let interval = DataType::DayTimeInterval {
+ start_field: 0,
+ end_field: 3,
+ };
+ let schema = DataType::Struct {
+ fields: vec![
+ field("interval", interval.clone()),
+ field(
+ "items",
+ DataType::Array {
+ element_type: Box::new(interval.clone()),
+ contains_null: true,
+ },
+ ),
+ field(
+ "nested",
+ DataType::Struct {
+ fields: vec![field("leaf", interval.clone())],
+ },
+ ),
+ field(
+ "mapping",
+ DataType::Map {
+ key_type: Box::new(DataType::String {
+ collation: "UTF8_BINARY".into(),
+ }),
+ value_type: Box::new(interval),
+ value_contains_null: true,
+ },
+ ),
+ ],
+ };
+ let names = vec![
+ "interval".into(),
+ "items".into(),
+ "nested".into(),
+ "mapping".into(),
+ ];
+ let mut rows = vec![Row::new(names.clone(), vec![Value::Null; 4])];
+ for micros in [
+ i64::MIN,
+ i64::MIN + 1,
+ -172_800_000_001,
+ -1,
+ 0,
+ 1,
+ i64::MAX - 1,
+ i64::MAX,
+ ] {
+ let value = Value::DayTimeInterval(micros);
+ rows.push(Row::new(
+ names.clone(),
+ vec![
+ value.clone(),
+ Value::List(vec![Value::Null, value.clone()]),
+ Value::Struct(Row::new(vec!["leaf".into()],
vec![value.clone()]).into()),
+ Value::Map(vec![
+ (Value::String("k".into()), value),
+ (Value::String("null".into()), Value::Null),
+ ]),
+ ],
+ ));
+ }
+ let df = s.create_dataframe(rows.clone(), schema.clone()).unwrap();
+ let collected = df.collect().unwrap();
+ assert_eq!(collected, rows);
+ assert_eq!(
+ s.create_dataframe(collected, schema)
+ .unwrap()
+ .collect()
+ .unwrap(),
+ rows
+ );
+
+ let sql = s
+ .sql("SELECT INTERVAL '-2 00:00:00.000001' DAY TO SECOND AS interval")
+ .unwrap();
+ let collected = sql.collect().unwrap();
+ assert_eq!(
+ collected[0].get(0),
+ Some(&Value::DayTimeInterval(-172_800_000_001))
+ );
+ assert_eq!(
+ s.create_dataframe(collected.clone(), sql.schema().unwrap())
+ .unwrap()
+ .collect()
+ .unwrap(),
+ collected
+ );
+}
diff --git a/docs/architecture.md b/docs/architecture.md
index 425f131..d7f0ca5 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -79,9 +79,12 @@ flowchart LR
When you call an action such as `count()` or `show()`, the lazily-built plan is
serialized and sent to the server; results stream back as Arrow batches.
-Day-time intervals are returned as Python `datetime.timedelta` values,
including
-nested values and nulls. Creating DataFrames from `timedelta` inputs or
collected
-interval rows is not yet supported; see
[SPARK-60659](https://issues.apache.org/jira/browse/SPARK-60659).
+Day-time intervals use signed 64-bit microseconds in Rust
(`Value::DayTimeInterval`)
+and Python `datetime.timedelta` values. `createDataFrame` supports inferred and
+explicit interval schemas, including nested structs, lists, maps, and nulls.
+Collected rows can be passed back with their schema. As in PySpark, an `int`
or a
+`float` for an interval field is taken as microseconds, and a value outside
the signed
+64-bit microsecond range is an error.
```mermaid
sequenceDiagram
diff --git a/python/tests/interval_input_cases.py
b/python/tests/interval_input_cases.py
new file mode 100644
index 0000000..ee41a1d
--- /dev/null
+++ b/python/tests/interval_input_cases.py
@@ -0,0 +1,211 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""SPARK-60659: exact interval inputs and live Spark round trips."""
+
+from datetime import timedelta
+import os
+import unittest
+
+from pyspark.sql import Row, SparkSession, types as T
+from pyspark.errors import PySparkTypeError
+
+from _mock_connect_server import mock_connect_session, parse_message
+
+try:
+ import grpc
+except ImportError:
+ grpc = None
+
+
+# The bounds that PySpark takes: a timedelta of whole days, whose microseconds
must fit in a
+# signed 64-bit integer on their own, cannot go below -106751991 days.
+LOWEST, HIGHEST = -106751991 * 86400000000, 2**63 - 1
+
+INFERENCE_CONFS = {
+ "spark.sql.pyspark.inferNestedDictAsStruct.enabled": "false",
+ "spark.sql.pyspark.legacy.inferArrayTypeFromFirstElement.enabled": "false",
+ "spark.sql.pyspark.legacy.inferMapTypeFromFirstPair.enabled": "false",
+ "spark.sql.timestampType": "TIMESTAMP_LTZ",
+}
+
+
+class IntervalInputConversionTests(unittest.TestCase):
+ def test_overflow_is_checked_recursively_before_rpc(self):
+ spark = SparkSession.builder.remote("sc://127.0.0.1:1").create()
+ interval = T.DayTimeIntervalType()
+ for micros in (-(2**63) - 1, 2**63, -(2**63) - 86400000000):
+ value = timedelta(microseconds=micros)
+ for data, data_type in (
+ ([(value,)], interval),
+ ([([value],)], T.ArrayType(interval)),
+ ([({"k": value},)], T.MapType(T.StringType(), interval)),
+ ([(Row(leaf=value),)], T.StructType([T.StructField("leaf",
interval)])),
+ ):
+ with self.subTest(micros=micros, data=data):
+ schema = T.StructType([T.StructField("interval",
data_type)])
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame(data, schema)
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame([(value,)])
+ # A Python Row stores objects unchanged; native conversion starts
here.
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame([Row(interval=value)])
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame([value], T.DayTimeIntervalType())
+ for value in (timedelta.min, timedelta.max):
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame([(value,)])
+
+ def test_inference_rejects_mixed_interval_and_numeric_values(self):
+ spark = SparkSession.builder.remote("sc://127.0.0.1:1").create()
+ delta = timedelta(microseconds=-1)
+ with self.assertRaises(PySparkTypeError) as error:
+ spark.createDataFrame([(delta,), (1,)])
+ self.assertEqual(error.exception.getCondition(), "CANNOT_MERGE_TYPE")
+ self.assertEqual(error.exception.getMessageParameters(), {
+ "data_type1": "DayTimeIntervalType", "data_type2": "LongType"})
+
+ @unittest.skipIf(grpc is None, "grpcio is required")
+ def test_nested_inference_reads_config_but_does_not_execute(self):
+ delta = timedelta(microseconds=-1)
+ with mock_connect_session() as (spark, service):
+ service.confs = INFERENCE_CONFS
+ for data in ([([delta, 1],)], [({"a": delta, "b": 1},)]):
+ with self.subTest(data=data):
+ with self.assertRaises(PySparkTypeError) as error:
+ spark.createDataFrame(data)
+ self.assertEqual(error.exception.getCondition(),
"CANNOT_INFER_TYPE_FOR_FIELD")
+ self.assertEqual(error.exception.getMessageParameters(),
{"field_name": "_1"})
+ spark.createDataFrame([([],), ([None, delta],)])
+ spark.createDataFrame([({},), ({"k": delta},)])
+ for data in ([([None, timedelta.max],)], [({"k":
timedelta.min},)]):
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame(data)
+ self.assertEqual(service.execute_calls, 0)
+ self.assertEqual(set(service.read_confs), set(INFERENCE_CONFS))
+
+ def test_bounds_and_null_first_inputs_build_without_rpc(self):
+ spark = SparkSession.builder.remote("sc://127.0.0.1:1").create()
+ for schema in (None, ["interval"], T.StructType([
+ T.StructField("interval", T.DayTimeIntervalType())])):
+ spark.createDataFrame([(None,), (timedelta(microseconds=LOWEST),),
+ (timedelta(microseconds=HIGHEST),)], schema)
+ interval = T.DayTimeIntervalType()
+ arrays = T.StructType([T.StructField("a", T.ArrayType(interval))])
+ maps = T.StructType([T.StructField("m", T.MapType(T.StringType(),
interval))])
+ spark.createDataFrame([([],), ([None, timedelta(microseconds=-1)],)],
arrays)
+ spark.createDataFrame([(Row(x=None),),
(Row(x=timedelta(microseconds=1)),)])
+ spark.createDataFrame([({},), ({"k": timedelta(microseconds=-1)},)],
maps)
+ # As PySpark, an int for an interval field is taken as microseconds.
+ spark.createDataFrame([(1,)], T.StructType([T.StructField("interval",
interval)]))
+
+ @unittest.skipIf(grpc is None, "grpcio is required")
+ def test_pandas_object_intervals_encode_exact_microseconds(self):
+ try:
+ import pandas as pd
+ import pyarrow as pa
+ except ImportError:
+ self.skipTest("pandas and pyarrow are required")
+ values = [None, LOWEST, LOWEST + 1, -1, 0, 1, HIGHEST]
+ data = pd.DataFrame({"interval": pd.Series([
+ None if value is None else timedelta(microseconds=value) for value
in values
+ ], dtype=object)})
+ with mock_connect_session() as (spark, service):
+ spark.createDataFrame(data).collect()
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.to_df=18,
+ # ToDF.input=1, Relation.local_relation=11, LocalRelation.data=1.
+ message = parse_message(service.execute_requests[0])
+ for field in (3, 1, 18, 1, 11):
+ message = parse_message(message[field][0])
+ table = pa.ipc.open_stream(message[1][0]).read_all()
+ self.assertEqual(table.column(0).type, pa.duration("us"))
+ self.assertEqual(table.column(0).cast(pa.int64()).to_pylist(),
values)
+ for value in (timedelta.min, timedelta.max):
+ overflow = pd.DataFrame({"interval": pd.Series([value],
dtype=object)})
+ with self.assertRaises(OverflowError):
+ spark.createDataFrame(overflow)
+ self.assertEqual(service.execute_calls, 1)
+
+
[email protected](os.environ.get("SPARK_REMOTE"), "requires a real Spark
Connect server")
+class IntervalInputLiveTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.spark =
SparkSession.builder.remote(os.environ["SPARK_REMOTE"]).create()
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.spark.stop()
+
+ def test_scalar_inferred_explicit_and_collected_inputs(self):
+ values = [None, LOWEST, LOWEST + 1, -172800000001, -1000001, -1,
+ 0, 1, 1000001, 172800000001, HIGHEST - 1, HIGHEST]
+ data = [(None if v is None else timedelta(microseconds=v),) for v in
values]
+ for schema in (None, ["interval"], T.StructType([
+ T.StructField("interval", T.DayTimeIntervalType())])):
+ with self.subTest(schema=schema):
+ frame = self.spark.createDataFrame(data, schema)
+ rows = frame.collect()
+ self.assertIsInstance(frame.schema.fields[0].dataType,
T.DayTimeIntervalType)
+ self.assertEqual([r[0] for r in rows], [r[0] for r in data])
+ again = self.spark.createDataFrame(rows, frame.schema)
+ self.assertEqual([r[0] for r in again.collect()], [r[0] for r
in data])
+ # Check an independently computed Spark SQL interval as well.
+ frame = self.spark.sql("SELECT INTERVAL '-2 00:00:00.000001' DAY TO
SECOND AS interval")
+ rows = frame.collect()
+ self.assertEqual(rows[0][0], timedelta(microseconds=-172800000001))
+ self.assertEqual(self.spark.createDataFrame(rows,
frame.schema).collect(), rows)
+
+ def test_explicit_atomic_null_and_empty_inputs(self):
+ interval = T.DayTimeIntervalType()
+ data = [None, timedelta(microseconds=-1), timedelta(microseconds=1)]
+ self.assertEqual([r[0] for r in self.spark.createDataFrame(data,
interval).collect()], data)
+ nulls = self.spark.createDataFrame([None, None], interval).collect()
+ self.assertEqual([r[0] for r in nulls], [None, None])
+ self.assertEqual(self.spark.createDataFrame([], interval).collect(),
[])
+ # As PySpark, an int for an interval field is taken as microseconds.
+ schema = T.StructType([T.StructField("interval", interval)])
+ self.assertEqual(self.spark.createDataFrame([(1,)], schema).collect(),
+ [Row(interval=timedelta(microseconds=1))])
+
+ def test_nested_null_first_structs_lists_and_maps(self):
+ delta = timedelta(microseconds=-172800000001)
+ data = [(None, None, None), (Row(leaf=None), [], {}),
+ (Row(leaf=delta), [None, delta], {"k": delta, "null": None})]
+ interval = T.DayTimeIntervalType()
+ explicit = T.StructType([
+ T.StructField("s", T.StructType([T.StructField("leaf",
interval)])),
+ T.StructField("a", T.ArrayType(interval)),
+ T.StructField("m", T.MapType(T.StringType(), interval)),
+ ])
+ for schema in (["s", "a", "m"], explicit):
+ with self.subTest(schema=schema):
+ frame = self.spark.createDataFrame(data, schema)
+ rows = frame.collect()
+ self.assertEqual(rows[0].asDict(), {"s": None, "a": None, "m":
None})
+ self.assertIsNone(rows[1][0][0])
+ self.assertEqual(rows[1][1], [])
+ self.assertEqual(rows[1][2], {})
+ self.assertEqual(rows[2][0][0], delta)
+ self.assertEqual(rows[2][1], [None, delta])
+ self.assertEqual(rows[2][2], {"k": delta, "null": None})
+ self.assertEqual(self.spark.createDataFrame(rows,
frame.schema).collect(), rows)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index bcfaeec..b0d18b1 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -30,6 +30,7 @@ from pyspark.sql import Row, SparkSession, types as T
# Collected here so the existing offline CI suite exercises real Arrow results.
from day_time_interval_cases import DayTimeIntervalTests # noqa: F401
+from interval_input_cases import IntervalInputConversionTests # noqa: F401
def test_create_dataframe_input_names_offline():
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]