This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 271abe8 [SPARK-61063][CONNECT][PYTHON] Take NumPy arrays of array
fields and ints of date fields, and send nested structs with duplicate field
names, in createDataFrame of local rows as PySpark does
271abe8 is described below
commit 271abe8adce6d5827e23c96da809fbcc1b05050b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:27:36 2026 +0900
[SPARK-61063][CONNECT][PYTHON] Take NumPy arrays of array fields and ints
of date fields, and send nested structs with duplicate field names, in
createDataFrame of local rows as PySpark does
### What changes were proposed in this pull request?
`createDataFrame` of local rows converts as PySpark 4.3.0 does in three
cases where it differed:
- An array field takes any sequence, NumPy arrays included, when its
elements need no converter, as `LocalDataToArrowConversion._need_converter`
decides (`crates/pyspark-rs/src/infer.rs`). Arrays of strings or binaries,
which need one, still require a list or `array.array`.
- A date field takes an int, a NumPy int, bool or float as days since the
epoch, as pyarrow does, with pyarrow's errors for a Python bool, out-of-range
values and other objects.
- The Arrow schema of a nested struct numbers duplicate field names (`a, a`
becomes `a_0, a_1`), as PySpark's `_dedup_names` does, while the relation's
schema keeps the names (`crates/spark-connect/src/session.rs`).
### Why are the changes needed?
[SPARK-61063](https://issues.apache.org/jira/browse/SPARK-61063): against a
local Spark 4.3.0 server:
| Input | Current master | This change and PySpark 4.3.0 |
| --- | --- | --- |
| `[(np.array([1, 2]),)]` into `array<bigint>` (also `array<double>`,
inside a struct) | `AssertionError` | `[Row(a=[1, 2])]` |
| `[(1,), (-1,)]` into `date` | `ValueError: Could not convert 1 with type
int` | `1970-01-02`, `1969-12-31` |
| `[(Row("a", "a")(1, 2),)]` | the server cannot read the IPC |
`[Row(_1=Row(a=1, a=2))]` |
### Does this PR introduce _any_ user-facing change?
Yes, these inputs convert as in PySpark.
### How was this patch tested?
- New offline tests in `python/tests/test_dropin_offline.py` that check the
IPC of each case against a mock server, a live test in
`python/tests/test_dropin_e2e.py` for the nested duplicate names, and a unit
test of the name numbering.
- The offline suite (930 passed, 8 skipped), and the lib tests of
`apache-spark-connect`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #254 from HyukjinKwon/SPARK-61063-local-rows.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/infer.rs | 149 +++++++++++++++++++++++++++++++-----
crates/spark-connect/src/session.rs | 58 ++++++++++----
python/tests/test_dropin_e2e.py | 13 ++++
python/tests/test_dropin_offline.py | 114 +++++++++++++++++++++++++++
4 files changed, 301 insertions(+), 33 deletions(-)
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index f62bd61..3ed7d6d 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -836,6 +836,15 @@ fn arrow_type_name(obj: &Bound<'_, PyAny>) ->
PyResult<String> {
})
}
+/// An error of PyArrow, of `class` with `message`, as its conversions raise
it.
+fn pyarrow_error(py: Python<'_>, class: &str, message: String) ->
PyResult<PyErr> {
+ let error = py
+ .import(intern!(py, "pyarrow"))?
+ .getattr(class)?
+ .call1((message,))?;
+ Ok(PyErr::from_value(error))
+}
+
/// The error of a value that PyArrow cannot convert to the type it is named
by.
fn arrow_invalid(obj: &Bound<'_, PyAny>, arrow_type: &str) -> PyResult<PyErr> {
Ok(PyValueError::new_err(format!(
@@ -960,12 +969,28 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
element_type,
contains_null,
} => {
- if !obj.is_instance_of::<PyList>()
- && !obj.is_instance(ARRAY.import(py, "array",
"array")?.as_any())?
- {
- return Err(assertion());
+ // As PySpark: an array whose elements need a converter takes a
list or an
+ // array.array, and another array is taken as it is, by PyArrow,
which takes any
+ // sequence, such as a tuple or a NumPy array of numbers or bools.
+ if needs_converter(dt, nullable) {
+ if !obj.is_instance_of::<PyList>()
+ && !obj.is_instance(ARRAY.import(py, "array",
"array")?.as_any())?
+ {
+ return Err(assertion());
+ }
+ } else if obj.is_instance_of::<PyDict>() ||
obj.try_iter().is_err() {
+ return Err(pyarrow_error(
+ py,
+ "ArrowTypeError",
+ format!(
+ "Could not convert {} with type {}: was not a sequence
or recognized \
+ null for conversion to list type",
+ obj.repr()?,
+ arrow_type_name(obj)?
+ ),
+ )?);
}
- let mut items = Vec::with_capacity(obj.len()?);
+ let mut items = Vec::with_capacity(obj.len().unwrap_or(0));
for item in obj.try_iter()? {
items.push(convert(&item?, element_type, *contains_null)?);
}
@@ -1093,14 +1118,7 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
})
}
DataType::DayTimeInterval { .. } =>
Ok(Value::DayTimeInterval(duration_micros(obj)?)),
- DataType::Date => {
- // PyArrow takes a date, or the date of a datetime.
- if !obj.is_instance(DATE.import(py, "datetime",
"date")?.as_any())? {
- return Err(arrow_invalid(obj, "date32")?);
- }
- let ordinal: i64 = obj.call_method0(intern!(py,
"toordinal"))?.extract()?;
- Ok(Value::Date((ordinal - 719_163) as i32))
- }
+ DataType::Date => Ok(Value::Date(date_days(obj)?)),
// As PyArrow's `time64[ns]`, to which PySpark passes the value as it
is: a time, and
// the nanoseconds of an int, or of a float, truncated.
DataType::Time { .. } => {
@@ -1178,6 +1196,101 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
}
}
+/// The days since the epoch of a value of a date column, as PyArrow converts
it to date32: a
+/// date, or the date of a datetime; an int, or a float truncated, as days;
and a NumPy bool or
+/// number as the value it stands for. Another value raises PyArrow's error.
+fn date_days(obj: &Bound<'_, PyAny>) -> PyResult<i32> {
+ let py = obj.py();
+ if obj.is_instance(DATE.import(py, "datetime", "date")?.as_any())? {
+ let ordinal: i64 = obj.call_method0(intern!(py,
"toordinal"))?.extract()?;
+ return Ok((ordinal - 719_163) as i32);
+ }
+ if obj.is_instance_of::<PyBool>() {
+ return Err(pyarrow_error(
+ py,
+ "ArrowTypeError",
+ "Expected integer, got bool".into(),
+ )?);
+ }
+ let kind = if obj.is_instance_of::<PyInt>() {
+ Some(NumpyScalar::Integer)
+ } else if obj.is_instance_of::<PyFloat>() {
+ Some(NumpyScalar::Floating)
+ } else {
+ numpy_scalar(obj)?
+ };
+ let days: i64 = match kind {
+ Some(NumpyScalar::Bool) => i64::from(obj.is_truthy()?),
+ Some(NumpyScalar::Integer) => obj.extract()?,
+ Some(NumpyScalar::Floating) => {
+ let value: f64 = obj.extract()?;
+ if value.is_nan() {
+ return Err(PyValueError::new_err("cannot convert float NaN to
integer"));
+ }
+ if value.is_infinite() {
+ return Err(PyOverflowError::new_err(
+ "cannot convert float infinity to integer",
+ ));
+ }
+ let truncated = value.trunc();
+ if
!(-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&truncated)
{
+ return Err(PyOverflowError::new_err(
+ "Python int too large to convert to C long",
+ ));
+ }
+ truncated as i64
+ }
+ None => {
+ return Err(pyarrow_error(
+ py,
+ "ArrowTypeError",
+ format!(
+ "object of type {} cannot be converted to int",
+ obj.get_type().repr()?
+ ),
+ )?)
+ }
+ };
+ // The days of PyArrow's dates: from 0001-01-01 to 9999-12-31, and a
larger magnitude is
+ // out of the range of a Python timedelta.
+ if days.unsigned_abs() > 999_999_999 {
+ return Err(PyOverflowError::new_err(format!(
+ "days={days}; must have magnitude <= 999999999"
+ )));
+ }
+ if !(-719_162..=2_932_896).contains(&days) {
+ return Err(PyOverflowError::new_err("date value out of range"));
+ }
+ Ok(days as i32)
+}
+
+/// Whether PySpark's `LocalDataToArrowConversion._need_converter` gives a
value of `dt` a
+/// converter, for a value of `nullable`: the values that PyArrow takes as
they are do not.
+fn needs_converter(dt: &DataType, nullable: bool) -> bool {
+ if !nullable {
+ return true;
+ }
+ match dt {
+ DataType::Null
+ | DataType::Struct { .. }
+ | DataType::Map { .. }
+ | DataType::Binary
+ | DataType::Timestamp
+ | DataType::TimestampNtz
+ | DataType::Decimal { .. }
+ | DataType::String { .. }
+ | DataType::Udt { .. }
+ | DataType::Variant
+ | DataType::Geometry { .. }
+ | DataType::Geography { .. } => true,
+ DataType::Array {
+ element_type,
+ contains_null,
+ } => needs_converter(element_type, *contains_null),
+ _ => false,
+ }
+}
+
/// The nanoseconds since midnight of a `datetime.time`, as PySpark's
`TimeType.toInternal`.
pub(crate) fn time_nanos(obj: &Bound<'_, PyAny>) -> PyResult<i64> {
let py = obj.py();
@@ -1258,15 +1371,9 @@ fn duration_micros(obj: &Bound<'_, PyAny>) ->
PyResult<i64> {
if obj.is_instance(TIMEDELTA.import(py, "datetime",
"timedelta")?.as_any())? {
return timedelta_micros(obj);
}
- let pyarrow_error = |class: &str, message: String| -> PyResult<PyErr> {
- let error = py
- .import(intern!(py, "pyarrow"))?
- .getattr(class)?
- .call1((message,))?;
- Ok(PyErr::from_value(error))
- };
if obj.is_instance_of::<PyBool>() {
return Err(pyarrow_error(
+ py,
"ArrowTypeError",
"Expected integer, got bool".into(),
)?);
@@ -1277,6 +1384,7 @@ fn duration_micros(obj: &Bound<'_, PyAny>) ->
PyResult<i64> {
(obj.getattr(intern!(py, "dtype"))?,),
)?;
return Err(pyarrow_error(
+ py,
"ArrowNotImplementedError",
format!("Expected np.timedelta64 but got: {}", arrow_type.str()?),
)?);
@@ -1302,6 +1410,7 @@ fn duration_micros(obj: &Bound<'_, PyAny>) ->
PyResult<i64> {
return Ok(truncated as i64);
}
Err(pyarrow_error(
+ py,
"ArrowTypeError",
format!(
"object of type {} cannot be converted to int",
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index c846c66..e75c0c6 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -1048,6 +1048,30 @@ fn schema_to_arrow_fields(schema: &DataType) ->
Result<Vec<arrow::datatypes::Fie
}
}
+/// The names of nested struct fields as PySpark's `_dedup_names` gives them:
a name that occurs
+/// more than once is numbered per occurrence, as `name_0`, `name_1`, and
others are kept.
+fn dedup_field_names<'a>(names: impl IntoIterator<Item = &'a str>) ->
Vec<String> {
+ let names: Vec<&str> = names.into_iter().collect();
+ let mut counts: std::collections::HashMap<&str, usize> =
std::collections::HashMap::new();
+ for name in &names {
+ *counts.entry(*name).or_insert(0) += 1;
+ }
+ let mut next: std::collections::HashMap<&str, usize> =
std::collections::HashMap::new();
+ names
+ .iter()
+ .map(|&name| {
+ if counts[name] > 1 {
+ let i = next.entry(name).or_insert(0);
+ let deduped = format!("{name}_{i}");
+ *i += 1;
+ deduped
+ } else {
+ name.to_string()
+ }
+ })
+ .collect()
+}
+
/// Map a Spark `DataType` to its Arrow `DataType`, recursively
(array/map/struct included).
fn datatype_to_arrow(dt: &DataType) -> Result<arrow::datatypes::DataType> {
use arrow::datatypes::{DataType as A, Field, Fields, TimeUnit};
@@ -1104,12 +1128,17 @@ fn datatype_to_arrow(dt: &DataType) ->
Result<arrow::datatypes::DataType> {
false,
)
}
+ // The names of nested fields are deduplicated, as PySpark's
`_deduplicate_field_names`
+ // does for the Arrow schema it sends. The server cannot read
duplicate names, while the
+ // schema of the relation keeps the names of the fields.
DataType::Struct { fields } => {
+ let names = dedup_field_names(fields.iter().map(|field|
field.name.as_str()));
let f: Fields = fields
.iter()
- .map(|field| {
+ .zip(names)
+ .map(|(field, name)| {
Ok(Field::new(
- &field.name,
+ name,
datatype_to_arrow(&field.data_type)?,
field.nullable,
))
@@ -1466,17 +1495,11 @@ fn values_to_arrow(values: &[&Value], dt: &DataType) ->
Result<Arc<dyn arrow::ar
.enumerate()
.map(|(i, f)| values_to_arrow(&cols[i], &f.data_type))
.collect::<Result<_>>()?;
- let afields: Fields = fields
- .iter()
- .map(|f| {
- Ok(Field::new(
- &f.name,
- datatype_to_arrow(&f.data_type)?,
- f.nullable,
- ))
- })
- .collect::<Result<Vec<_>>>()?
- .into();
+ // The Arrow fields of the struct, with the deduplicated names of
`datatype_to_arrow`,
+ // which the column's type has.
+ let arrow::datatypes::DataType::Struct(afields) =
datatype_to_arrow(dt)? else {
+ unreachable!("a struct is a struct")
+ };
// A struct without fields takes its length from the values.
StructArray::try_new_with_length(
afields,
@@ -2181,6 +2204,15 @@ mod tests {
assert_eq!(actual.column(6).null_count(), 3);
}
+ #[test]
+ fn nested_duplicate_field_names_are_numbered_per_occurrence_as_pyspark() {
+ assert_eq!(dedup_field_names(["a", "b"]), vec!["a", "b"]);
+ assert_eq!(
+ dedup_field_names(["a", "a", "b", "a"]),
+ vec!["a_0", "a_1", "b", "a_2"]
+ );
+ }
+
#[test]
fn struct_arrow_encoding_preserves_lookup_map_and_null_semantics() {
use arrow::array::{Array, Int16Array, Int32Array, Int64Array,
StructArray};
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 4ae90d8..89bc02c 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -175,6 +175,19 @@ def
test_get_or_create_returns_the_session_with_the_options_as_pyspark(session):
session.conf.set(key, previous)
+def test_nested_duplicate_field_names_of_local_rows_reach_the_server(session):
+ # As PySpark: the server reads the Arrow data of nested fields with
deduplicated names,
+ # and the schema of the relation keeps the names, so the rows come back as
they were.
+ R = Row("a", "a")
+ df = session.createDataFrame([(R(1, 2),)])
+ assert df.schema.simpleString() == "struct<_1:struct<a:bigint,a:bigint>>"
+ assert tuple(df.collect()[0]._1) == (1, 2)
+ df = session.createDataFrame([((1, 2),)], "s struct<a:bigint,a:bigint>")
+ assert tuple(df.collect()[0].s) == (1, 2)
+ df = session.createDataFrame([([(1, 2)],)], "s
array<struct<a:bigint,a:bigint>>")
+ assert [tuple(item) for item in df.collect()[0].s] == [(1, 2)]
+
+
def test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
df = session.range(3)
assert repr(df) == "DataFrame[id: bigint]"
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 9d7152f..3d0f7aa 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7151,6 +7151,120 @@ def test_arrow_input_large_upload_live_server():
spark.stop()
+# ---------------------------------------------------------------------------
local rows of NumPy arrays, ints for dates and nested duplicate names
+
+def _local_rows_relation_of(service, df):
+ # The LocalRelation of a DataFrame of local rows, as the server receives
it: its Arrow IPC
+ # data (LocalRelation.data=1), and its schema JSON
(LocalRelation.schema=2).
+ import pyarrow as pa
+
+ df.toArrow()
+ request = parse_message(service.execute_requests[-1])
+ relation = parse_message(parse_message(request[3][0])[1][0])
+ while 18 in relation: # Relation.to_df -> ToDF.input.
+ relation = parse_message(parse_message(relation[18][0])[1][0])
+ local = parse_message(relation[11][0])
+ return pa.ipc.open_stream(local[1][0]).read_all(), json.loads(local[2][0])
+
+
+def test_numpy_arrays_in_array_fields_convert_as_pyspark(mock_connect):
+ # As PySpark: an array whose elements need no converter (numbers and
bools) is taken as
+ # PyArrow takes it, so a NumPy array is an array of its values. An array
whose elements
+ # need one (strings) must be a list or an array.array, or PySpark asserts.
+ from pyspark.sql import types as T
+
+ np = pytest.importorskip("numpy")
+ spark, service = mock_connect
+ table, _ = _local_rows_relation_of(
+ service, spark.createDataFrame(
+ [(np.array([1, 2]), np.array([1.5, 2.0]), np.array([True,
False]))],
+ "a array<bigint>, b array<double>, c array<boolean>"))
+ assert table.column("a").to_pylist() == [[1, 2]]
+ assert table.column("b").to_pylist() == [[1.5, 2.0]]
+ assert table.column("c").to_pylist() == [[True, False]]
+
+ long_array = T.StructType([T.StructField("a", T.ArrayType(T.LongType()))])
+ table, _ = _local_rows_relation_of(
+ service, spark.createDataFrame([(np.array([1, 2]),)], long_array))
+ assert table.column("a").to_pylist() == [[1, 2]]
+ table, _ = _local_rows_relation_of(
+ service, spark.createDataFrame([((np.array([1, 2]),),)], "s
struct<a:array<bigint>>"))
+ assert table.column("s").to_pylist() == [{"a": [1, 2]}]
+
+ for data, schema in [
+ ([(np.array(["a", "b"]),)], "a array<string>"),
+ ([(("a", "b"),)], "a array<string>"),
+ ([(np.array([1, 2]),)], T.StructType([T.StructField(
+ "a", T.ArrayType(T.LongType(), containsNull=False))])),
+ ]:
+ # PySpark's assertion, which has no message.
+ with pytest.raises(AssertionError) as error:
+ spark.createDataFrame(data, schema)
+ assert str(error.value) == ""
+
+
+def test_ints_of_date_fields_convert_as_pyarrow(mock_connect):
+ # As PySpark: PyArrow takes an int, or a float truncated, as days since
the epoch, and a
+ # NumPy bool or number as the value it stands for. A Python bool, a string
and the days
+ # out of the calendar range raise PyArrow's errors.
+ import re
+
+ import pyarrow as pa
+
+ np = pytest.importorskip("numpy")
+ spark, service = mock_connect
+ table, _ = _local_rows_relation_of(
+ service, spark.createDataFrame([(1,), (-1,), (None,)], "a date"))
+ assert table.schema.field(0).type == pa.date32()
+ assert table.column(0).cast(pa.int32()).to_pylist() == [1, -1, None]
+ table, _ = _local_rows_relation_of(service, spark.createDataFrame(
+ [(np.int64(3),), (1.5,), (-1.5,), (np.bool_(True),), (2932896,),
(-719162,)], "a date"))
+ assert table.column(0).cast(pa.int32()).to_pylist() == [3, 1, -1, 1,
2932896, -719162]
+
+ for value, error, message in (
+ (True, pa.ArrowTypeError, "Expected integer, got bool"),
+ ("x", pa.ArrowTypeError, "object of type <class 'str'> cannot be
converted to int"),
+ (float("nan"), ValueError, "cannot convert float NaN to integer"),
+ (2932897, OverflowError, "date value out of range"),
+ (-719163, OverflowError, "date value out of range"),
+ (10**9, OverflowError, "days=1000000000; must have magnitude <=
999999999"),
+ (2**70, OverflowError, "Python int too large to convert to C long"),
+ ):
+ with pytest.raises(error, match="^" + re.escape(message) + "$"):
+ spark.createDataFrame([(value,)], "a date")
+
+
+def
test_nested_duplicate_field_names_are_deduplicated_in_the_arrow_schema(mock_connect):
+ # As PySpark's _deduplicate_field_names: the Arrow schema of nested struct
fields numbers
+ # duplicate names, which the server cannot read, while the schema of the
relation keeps
+ # the names. A top-level duplicate name is not renamed.
+ import pyarrow as pa
+ from pyspark.sql import Row
+ from pyspark.sql import types as T
+
+ spark, service = mock_connect
+ R = Row("a", "a")
+ table, schema = _local_rows_relation_of(service,
spark.createDataFrame([(R(1, 2),)]))
+ assert table.schema.field(0).type == pa.struct([("a_0", pa.int64()),
("a_1", pa.int64())])
+ assert table.column(0).to_pylist() == [{"a_0": 1, "a_1": 2}]
+ assert [f["name"] for f in schema["fields"][0]["type"]["fields"]] == ["a",
"a"]
+
+ explicit = T.StructType([T.StructField("s", T.StructType([
+ T.StructField("a", T.LongType()), T.StructField("a", T.LongType())]))])
+ table, schema = _local_rows_relation_of(service,
spark.createDataFrame([((1, 2),)], explicit))
+ assert table.column("s").to_pylist() == [{"a_0": 1, "a_1": 2}]
+ assert [f["name"] for f in schema["fields"][0]["type"]["fields"]] == ["a",
"a"]
+
+ table, _ = _local_rows_relation_of(service, spark.createDataFrame(
+ [([(1, 2)],)], "s array<struct<a:bigint,a:bigint>>"))
+ assert table.column("s").to_pylist() == [[{"a_0": 1, "a_1": 2}]]
+
+ table, _ = _local_rows_relation_of(service, spark.createDataFrame(
+ [(1, 2)], "a bigint, a bigint"))
+ assert table.schema.names == ["a", "a"]
+ assert table.column(1).to_pylist() == [2]
+
+
# ------------------------------------------------------- DataFrame completion
def _completion_varint(value):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]