This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new a567fc6 [SPARK-61103][CONNECT][PYTHON] Create a DataFrame of a numpy
array in createDataFrame in the Python client as PySpark does
a567fc6 is described below
commit a567fc69f26ac22b252ef5a5e39bad52c1646571
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 17:50:12 2026 +0900
[SPARK-61103][CONNECT][PYTHON] Create a DataFrame of a numpy array in
createDataFrame in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61103](https://issues.apache.org/jira/browse/SPARK-61103):
`spark.createDataFrame` now takes a numpy array as PySpark Connect's does.
As PySpark Connect's, an array of 1 or 2 dimensions becomes an Arrow table
with a column for each column of the array. The table is sent with the schema,
if there is one, and the server casts to it. The columns are named:
- by a list or tuple of names, if one is given
- `value`, for a 1-dimensional array or an array of one column
- `_1`, `_2`, ... otherwise
It raises:
- `INVALID_NDARRAY_DIMENSION` for an array of another dimension
- `AXIS_LENGTH_MISMATCH` when the names, or a schema, have another number
of columns
An empty array is empty data, as an empty list is.
The Arrow table is built by `pa.array` of each column, as in PySpark. The
values, the types and the errors of pyarrow and the server are therefore the
same. Examples are datetimes of `datetime64[us]` as `timestamp_ntz`,
`UNSUPPORTED_ARROWTYPE` for `uint64`, and `CAST_OVERFLOW` when the server casts
to a schema.
### Why are the changes needed?
The client took a numpy array as rows of Python values, whose types it
could not infer. `spark.createDataFrame(np.array([1, 2]))` raised
`CANNOT_INFER_TYPE_FOR_FIELD`, and so did an array of 3 dimensions.
### Does this PR introduce _any_ user-facing change?
Yes. `createDataFrame` of a numpy array creates the DataFrame that PySpark
4.3.0 creates.
### How was this patch tested?
- A new live test checks the schema and the rows of a 1- and a
2-dimensional array, of names and of a DDL schema, and the errors. It passes
with official PySpark 4.3.0 and fails without this change.
- A new offline test checks the Arrow data and the schema sent for each way
of naming the columns, and the errors.
- On a local server, 65 cases give the same results as official PySpark
4.3.0. They cover:
- arrays of 0 to 3 dimensions and of each dtype, including object, masked
and datetime arrays
- names and struct, DDL and atomic schemas
- empty arrays
- The lib tests of both crates (588 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2051 passed, 12 skipped), and the
live drop-in and Arrow suites (155 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #303 from HyukjinKwon/SPARK-61103-numpy-create-dataframe.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/pyarrow_import.rs | 23 ++++++----
crates/pyspark-rs/src/session.rs | 80 +++++++++++++++++++++++++++++++++
python/pyspark/sql/_arrow_input.py | 10 +++++
python/tests/test_dropin_e2e.py | 21 +++++++++
python/tests/test_dropin_offline.py | 48 ++++++++++++++++++++
5 files changed, 173 insertions(+), 9 deletions(-)
diff --git a/crates/pyspark-rs/src/pyarrow_import.rs
b/crates/pyspark-rs/src/pyarrow_import.rs
index e2fea5f..769181b 100644
--- a/crates/pyspark-rs/src/pyarrow_import.rs
+++ b/crates/pyspark-rs/src/pyarrow_import.rs
@@ -25,6 +25,19 @@ use spark_connect::session::SparkSession;
use crate::dataframe::PyDataFrame;
use crate::errors::ResultExt;
+/// The record batches of a pyarrow table, at least one, so that a table
without rows keeps its
+/// schema, as PySpark's IPC stream does, for the relation to carry it.
+pub(crate) fn table_batches(table: &Bound<'_, PyAny>) ->
PyResult<Vec<RecordBatch>> {
+ // PyArrow 10 readers support _export_to_c, even without PyCapsules.
+ let (mut batches, arrow_schema) =
+
Table::from_pyarrow_bound(&table.call_method0("to_reader")?)?.into_inner();
+ // A cast drops the rows of a zero-column table, leaving no batches.
+ if batches.is_empty() {
+ batches.push(RecordBatch::new_empty(arrow_schema));
+ }
+ Ok(batches)
+}
+
pub(crate) fn create_dataframe(
py: Python<'_>,
session: &SparkSession,
@@ -67,15 +80,7 @@ pub(crate) fn create_dataframe(
let batches = if empty {
Vec::new()
} else {
- // PyArrow 10 readers support _export_to_c, even without PyCapsules.
- let (mut batches, arrow_schema) =
-
Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)?.into_inner();
- // The cast drops the rows of a zero-column table, leaving no batches.
Keep its
- // schema, as PySpark's IPC stream does, so the relation still carries
one.
- if batches.is_empty() {
- batches.push(RecordBatch::new_empty(arrow_schema));
- }
- batches
+ table_batches(&normalized)?
};
let dataframe = py.detach(|| {
match options.as_ref() {
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 8007ee5..659ec20 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -706,6 +706,83 @@ impl PySparkSession {
))))
}
+ /// A DataFrame of a numpy array, as PySpark Connect's `createDataFrame`
builds it: an Arrow
+ /// table of a column of each column of the array, sent with the schema,
if any, for the
+ /// server to cast to. The columns are named by the given names, else
`value` for a
+ /// 1-dimensional array or an array of one column, else `_1`, `_2`, ...
+ ///
+ /// It raises `INVALID_NDARRAY_DIMENSION` for an array of another
dimension, and
+ /// `AXIS_LENGTH_MISMATCH` for names, or a schema, of another number of
columns. An empty
+ /// array is empty data, of the schema, if any.
+ fn create_from_ndarray(
+ &self,
+ py: Python<'_>,
+ data: &Bound<'_, PyAny>,
+ spec: Spec,
+ ) -> PyResult<PyDataFrame> {
+ let ndim: usize = data.getattr("ndim")?.extract()?;
+ if ndim != 1 && ndim != 2 {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "INVALID_NDARRAY_DIMENSION",
+ &[("dimensions", "1 or 2")],
+ ));
+ }
+ if data.len()? == 0 {
+ return self.create_from_local(py, PyList::empty(py).as_any(),
spec);
+ }
+ let width: usize = if ndim == 1 {
+ 1
+ } else {
+ data.getattr("shape")?.get_item(1)?.extract()?
+ };
+ let (names, num_cols, schema) = match spec {
+ Spec::Names(names) => (Some(names), None, None),
+ Spec::Struct(fields) => (None, Some(fields.len()),
Some(DataType::Struct { fields })),
+ Spec::Atomic(atomic) => (None, Some(1), Some(atomic)),
+ Spec::None => (None, None, None),
+ };
+ let names = names.unwrap_or_else(|| {
+ if width == 1 {
+ vec!["value".to_string()]
+ } else {
+ (1..=width).map(|i| format!("_{i}")).collect()
+ }
+ });
+ let axis_length_mismatch = |expected: usize| {
+ crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "AXIS_LENGTH_MISMATCH",
+ &[
+ ("expected_length", &expected.to_string()),
+ ("actual_length", &width.to_string()),
+ ],
+ )
+ };
+ if names.len() != width {
+ return Err(axis_length_mismatch(names.len()));
+ }
+ let table = py
+ .import("pyspark.sql._arrow_input")?
+ .getattr("ndarray_table")?
+ .call1((data, names))?;
+ if let Some(expected) = num_cols.filter(|&expected| expected != width)
{
+ return Err(axis_length_mismatch(expected));
+ }
+ let batches = crate::pyarrow_import::table_batches(&table)?;
+ let options = py.detach(||
self.session.arrow_input_options().to_pyerr())?;
+ let df = py
+ .detach(|| {
+ self.session
+ .create_dataframe_from_arrow_with_options(batches, schema,
&options)
+ .map_err(Box::new)
+ })
+ .to_pyerr()?;
+ Ok(PyDataFrame::new(df))
+ }
+
/// A DataFrame of a pandas DataFrame: its rows as lists with NaN and NaT
as None, typed
/// by the first non-null value of each column, and its column labels, as
strings, or the
/// given names, applied with `toDF`.
@@ -1182,6 +1259,9 @@ impl PySparkSession {
return crate::pyarrow_import::create_dataframe(py, &self.session,
data, schema);
}
let spec = schema_spec(py, &self.session, schema)?;
+ if is_loaded_instance(py, data, "numpy", "ndarray")? {
+ return self.create_from_ndarray(py, data, spec);
+ }
if is_loaded_instance(py, data, "pandas", "DataFrame")? {
// Guarded integer columns share the existing Arrow importer.
Schema
// validation above and unsupported pandas conversions keep
master's
diff --git a/python/pyspark/sql/_arrow_input.py
b/python/pyspark/sql/_arrow_input.py
index 0c5b337..8bcd2af 100644
--- a/python/pyspark/sql/_arrow_input.py
+++ b/python/pyspark/sql/_arrow_input.py
@@ -91,3 +91,13 @@ def prepare_table(table, schema, timestamp_type, timezone,
large_types):
if not explicit and names is None:
names = table.column_names
return normalized, schema if explicit else None, names
+
+
+def ndarray_table(data, names):
+ """The Arrow table of a numpy array of 1 or 2 dimensions, of a column of
each column of the
+ array, named `names`, as PySpark Connect's createDataFrame builds it."""
+ import pyarrow as pa
+
+ if data.ndim == 1:
+ return pa.Table.from_arrays([pa.array(data)], names)
+ return pa.Table.from_arrays([pa.array(data[::, i]) for i in range(0,
data.shape[1])], names)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 13cd21f..65db098 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2093,6 +2093,27 @@ def
test_window_order_by_a_column_sorts_nulls_first_as_pyspark(session):
assert [(r.o, r.rn) for r in rows] == expected
+def test_create_dataframe_of_a_numpy_array_as_pyspark(session):
+ # As PySpark Connect's: a column of each column of a 1- or 2-dimensional
array, named by the
+ # names, else `value` for one column, else `_1`, `_2`, ..., of the types
of the array, or
+ # cast to the schema by the server.
+ import numpy as np
+
+ df = session.createDataFrame(np.array([1, 2]))
+ assert df.schema.simpleString() == "struct<value:bigint>"
+ assert sorted(df.collect()) == [Row(value=1), Row(value=2)]
+ df = session.createDataFrame(np.array([[1.5, 2.5], [3.5, 4.5]]))
+ assert df.schema.simpleString() == "struct<_1:double,_2:double>"
+ assert session.createDataFrame(np.array([[1, 2]]), "a string, b
double").collect() == [
+ Row(a="1", b=2.0)]
+ assert session.createDataFrame(np.array(["x", "y"]), ["s"]).columns ==
["s"]
+ with pytest.raises(PySparkValueError) as e:
+ session.createDataFrame(np.zeros((2, 2, 2)))
+ assert e.value.getCondition() == "INVALID_NDARRAY_DIMENSION"
+ with pytest.raises(PySparkValueError) as e:
+ session.createDataFrame(np.array([[1, 2]]), ["x"])
+ assert e.value.getCondition() == "AXIS_LENGTH_MISMATCH"
+
def test_create_dataframe_takes_the_ddl_of_spark_as_pyspark(session):
# As PySpark Connect's, which parses a DDL schema on the server: nested
structs, arrays and
# maps of them, backtick-quoted names, and NOT NULL and COMMENT clauses.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b4c5b63..a1aa149 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6043,6 +6043,54 @@ def
test_create_dataframe_schema_is_names_a_struct_or_an_atomic_type(schema, arg
"arg_name": "schema", "expected_type": "list, None or StructType",
"arg_type": arg_type}
+def test_create_dataframe_of_a_numpy_array_as_pyspark(mock_connect):
+ # As PySpark Connect's: an Arrow table of a column of each column of a 1-
or 2-dimensional
+ # array, named by the names, else `value` for one column, else `_1`, `_2`,
..., sent with
+ # the schema, if any, for the server to cast to.
+ import numpy as np
+ import pyarrow as pa
+ from pyspark.errors import PySparkValueError
+
+ spark, service = mock_connect
+
+ def sent(df):
+ df.columns
+ # Relation.local_relation=11: data=1, an Arrow IPC stream, and
schema=2.
+ relation = _sent_relation(service.analyze_requests[-1], 11)
+ table = pa.ipc.open_stream(relation[1][0]).read_all()
+ return table.to_pydict(), [json.loads(s) for s in relation.get(2, [])]
+
+ assert sent(spark.createDataFrame(np.array([1, 2]))) == ({"value": [1,
2]}, [])
+ assert sent(spark.createDataFrame(np.array([[1, 2], [3, 4]]))) == (
+ {"_1": [1, 3], "_2": [2, 4]}, [])
+ assert sent(spark.createDataFrame(np.array([[1], [2]]), ("x",))) == ({"x":
[1, 2]}, [])
+ schema = T.StructType().add("a", T.IntegerType()).add("b", T.StringType())
+ assert sent(spark.createDataFrame(np.array([[1, 2]]), schema)) == (
+ {"_1": [1], "_2": [2]}, [json.loads(schema.json())])
+ assert sent(spark.createDataFrame(np.array([1.5]), T.DoubleType())) == (
+ {"value": [1.5]}, ["double"])
+ for data, schema, expected, actual in [
+ (np.array([1, 2]), ["x", "y"], "2", "1"),
+ (np.array([1, 2]), [], "0", "1"),
+ (np.array([[1, 2]]), ["x"], "1", "2"),
+ (np.array([[1, 2]]), T.IntegerType(), "1", "2"),
+ (np.array([1, 2]), T.StructType().add("a", T.IntegerType()).add("b",
T.LongType()),
+ "2", "1"),
+ ]:
+ with pytest.raises(PySparkValueError) as e:
+ spark.createDataFrame(data, schema)
+ assert e.value.getCondition() == "AXIS_LENGTH_MISMATCH"
+ assert e.value.getMessageParameters() == {
+ "expected_length": expected, "actual_length": actual}
+ for data in (np.zeros((2, 2, 2)), np.array(5)):
+ with pytest.raises(PySparkValueError) as e:
+ spark.createDataFrame(data)
+ assert e.value.getCondition() == "INVALID_NDARRAY_DIMENSION"
+ assert e.value.getMessageParameters() == {"dimensions": "1 or 2"}
+ with pytest.raises(PySparkValueError) as e:
+ spark.createDataFrame(np.array([]))
+ assert e.value.getCondition() == "CANNOT_INFER_EMPTY_SCHEMA"
+
def test_create_dataframe_of_a_dataframe_raises_as_pyspark():
from pyspark.errors import PySparkTypeError
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]