This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a567fc6  [SPARK-61103][CONNECT][PYTHON] Create a DataFrame of a numpy 
array in createDataFrame in the Python client as PySpark does
a567fc6 is described below

commit a567fc69f26ac22b252ef5a5e39bad52c1646571
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 17:50:12 2026 +0900

    [SPARK-61103][CONNECT][PYTHON] Create a DataFrame of a numpy array in 
createDataFrame in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61103](https://issues.apache.org/jira/browse/SPARK-61103): 
`spark.createDataFrame` now takes a numpy array as PySpark Connect's does.
    
    As PySpark Connect's, an array of 1 or 2 dimensions becomes an Arrow table 
with a column for each column of the array. The table is sent with the schema, 
if there is one, and the server casts to it. The columns are named:
    - by a list or tuple of names, if one is given
    - `value`, for a 1-dimensional array or an array of one column
    - `_1`, `_2`, ... otherwise
    
    It raises:
    - `INVALID_NDARRAY_DIMENSION` for an array of another dimension
    - `AXIS_LENGTH_MISMATCH` when the names, or a schema, have another number 
of columns
    
    An empty array is empty data, as an empty list is.
    
    The Arrow table is built by `pa.array` of each column, as in PySpark. The 
values, the types and the errors of pyarrow and the server are therefore the 
same. Examples are datetimes of `datetime64[us]` as `timestamp_ntz`, 
`UNSUPPORTED_ARROWTYPE` for `uint64`, and `CAST_OVERFLOW` when the server casts 
to a schema.
    
    ### Why are the changes needed?
    
    The client took a numpy array as rows of Python values, whose types it 
could not infer. `spark.createDataFrame(np.array([1, 2]))` raised 
`CANNOT_INFER_TYPE_FOR_FIELD`, and so did an array of 3 dimensions.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `createDataFrame` of a numpy array creates the DataFrame that PySpark 
4.3.0 creates.
    
    ### How was this patch tested?
    
    - A new live test checks the schema and the rows of a 1- and a 
2-dimensional array, of names and of a DDL schema, and the errors. It passes 
with official PySpark 4.3.0 and fails without this change.
    - A new offline test checks the Arrow data and the schema sent for each way 
of naming the columns, and the errors.
    - On a local server, 65 cases give the same results as official PySpark 
4.3.0. They cover:
      - arrays of 0 to 3 dimensions and of each dtype, including object, masked 
and datetime arrays
      - names and struct, DDL and atomic schemas
      - empty arrays
    - The lib tests of both crates (588 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2051 passed, 12 skipped), and the 
live drop-in and Arrow suites (155 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #303 from HyukjinKwon/SPARK-61103-numpy-create-dataframe.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/pyarrow_import.rs | 23 ++++++----
 crates/pyspark-rs/src/session.rs        | 80 +++++++++++++++++++++++++++++++++
 python/pyspark/sql/_arrow_input.py      | 10 +++++
 python/tests/test_dropin_e2e.py         | 21 +++++++++
 python/tests/test_dropin_offline.py     | 48 ++++++++++++++++++++
 5 files changed, 173 insertions(+), 9 deletions(-)

diff --git a/crates/pyspark-rs/src/pyarrow_import.rs 
b/crates/pyspark-rs/src/pyarrow_import.rs
index e2fea5f..769181b 100644
--- a/crates/pyspark-rs/src/pyarrow_import.rs
+++ b/crates/pyspark-rs/src/pyarrow_import.rs
@@ -25,6 +25,19 @@ use spark_connect::session::SparkSession;
 use crate::dataframe::PyDataFrame;
 use crate::errors::ResultExt;
 
+/// The record batches of a pyarrow table, at least one, so that a table 
without rows keeps its
+/// schema, as PySpark's IPC stream does, for the relation to carry it.
+pub(crate) fn table_batches(table: &Bound<'_, PyAny>) -> 
PyResult<Vec<RecordBatch>> {
+    // PyArrow 10 readers support _export_to_c, even without PyCapsules.
+    let (mut batches, arrow_schema) =
+        
Table::from_pyarrow_bound(&table.call_method0("to_reader")?)?.into_inner();
+    // A cast drops the rows of a zero-column table, leaving no batches.
+    if batches.is_empty() {
+        batches.push(RecordBatch::new_empty(arrow_schema));
+    }
+    Ok(batches)
+}
+
 pub(crate) fn create_dataframe(
     py: Python<'_>,
     session: &SparkSession,
@@ -67,15 +80,7 @@ pub(crate) fn create_dataframe(
     let batches = if empty {
         Vec::new()
     } else {
-        // PyArrow 10 readers support _export_to_c, even without PyCapsules.
-        let (mut batches, arrow_schema) =
-            
Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)?.into_inner();
-        // The cast drops the rows of a zero-column table, leaving no batches. 
Keep its
-        // schema, as PySpark's IPC stream does, so the relation still carries 
one.
-        if batches.is_empty() {
-            batches.push(RecordBatch::new_empty(arrow_schema));
-        }
-        batches
+        table_batches(&normalized)?
     };
     let dataframe = py.detach(|| {
         match options.as_ref() {
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 8007ee5..659ec20 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -706,6 +706,83 @@ impl PySparkSession {
         ))))
     }
 
+    /// A DataFrame of a numpy array, as PySpark Connect's `createDataFrame` 
builds it: an Arrow
+    /// table of a column of each column of the array, sent with the schema, 
if any, for the
+    /// server to cast to. The columns are named by the given names, else 
`value` for a
+    /// 1-dimensional array or an array of one column, else `_1`, `_2`, ...
+    ///
+    /// It raises `INVALID_NDARRAY_DIMENSION` for an array of another 
dimension, and
+    /// `AXIS_LENGTH_MISMATCH` for names, or a schema, of another number of 
columns. An empty
+    /// array is empty data, of the schema, if any.
+    fn create_from_ndarray(
+        &self,
+        py: Python<'_>,
+        data: &Bound<'_, PyAny>,
+        spec: Spec,
+    ) -> PyResult<PyDataFrame> {
+        let ndim: usize = data.getattr("ndim")?.extract()?;
+        if ndim != 1 && ndim != 2 {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "INVALID_NDARRAY_DIMENSION",
+                &[("dimensions", "1 or 2")],
+            ));
+        }
+        if data.len()? == 0 {
+            return self.create_from_local(py, PyList::empty(py).as_any(), 
spec);
+        }
+        let width: usize = if ndim == 1 {
+            1
+        } else {
+            data.getattr("shape")?.get_item(1)?.extract()?
+        };
+        let (names, num_cols, schema) = match spec {
+            Spec::Names(names) => (Some(names), None, None),
+            Spec::Struct(fields) => (None, Some(fields.len()), 
Some(DataType::Struct { fields })),
+            Spec::Atomic(atomic) => (None, Some(1), Some(atomic)),
+            Spec::None => (None, None, None),
+        };
+        let names = names.unwrap_or_else(|| {
+            if width == 1 {
+                vec!["value".to_string()]
+            } else {
+                (1..=width).map(|i| format!("_{i}")).collect()
+            }
+        });
+        let axis_length_mismatch = |expected: usize| {
+            crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "AXIS_LENGTH_MISMATCH",
+                &[
+                    ("expected_length", &expected.to_string()),
+                    ("actual_length", &width.to_string()),
+                ],
+            )
+        };
+        if names.len() != width {
+            return Err(axis_length_mismatch(names.len()));
+        }
+        let table = py
+            .import("pyspark.sql._arrow_input")?
+            .getattr("ndarray_table")?
+            .call1((data, names))?;
+        if let Some(expected) = num_cols.filter(|&expected| expected != width) 
{
+            return Err(axis_length_mismatch(expected));
+        }
+        let batches = crate::pyarrow_import::table_batches(&table)?;
+        let options = py.detach(|| 
self.session.arrow_input_options().to_pyerr())?;
+        let df = py
+            .detach(|| {
+                self.session
+                    .create_dataframe_from_arrow_with_options(batches, schema, 
&options)
+                    .map_err(Box::new)
+            })
+            .to_pyerr()?;
+        Ok(PyDataFrame::new(df))
+    }
+
     /// A DataFrame of a pandas DataFrame: its rows as lists with NaN and NaT 
as None, typed
     /// by the first non-null value of each column, and its column labels, as 
strings, or the
     /// given names, applied with `toDF`.
@@ -1182,6 +1259,9 @@ impl PySparkSession {
             return crate::pyarrow_import::create_dataframe(py, &self.session, 
data, schema);
         }
         let spec = schema_spec(py, &self.session, schema)?;
+        if is_loaded_instance(py, data, "numpy", "ndarray")? {
+            return self.create_from_ndarray(py, data, spec);
+        }
         if is_loaded_instance(py, data, "pandas", "DataFrame")? {
             // Guarded integer columns share the existing Arrow importer. 
Schema
             // validation above and unsupported pandas conversions keep 
master's
diff --git a/python/pyspark/sql/_arrow_input.py 
b/python/pyspark/sql/_arrow_input.py
index 0c5b337..8bcd2af 100644
--- a/python/pyspark/sql/_arrow_input.py
+++ b/python/pyspark/sql/_arrow_input.py
@@ -91,3 +91,13 @@ def prepare_table(table, schema, timestamp_type, timezone, 
large_types):
     if not explicit and names is None:
         names = table.column_names
     return normalized, schema if explicit else None, names
+
+
+def ndarray_table(data, names):
+    """The Arrow table of a numpy array of 1 or 2 dimensions, of a column of 
each column of the
+    array, named `names`, as PySpark Connect's createDataFrame builds it."""
+    import pyarrow as pa
+
+    if data.ndim == 1:
+        return pa.Table.from_arrays([pa.array(data)], names)
+    return pa.Table.from_arrays([pa.array(data[::, i]) for i in range(0, 
data.shape[1])], names)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 13cd21f..65db098 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2093,6 +2093,27 @@ def 
test_window_order_by_a_column_sorts_nulls_first_as_pyspark(session):
         assert [(r.o, r.rn) for r in rows] == expected
 
 
+def test_create_dataframe_of_a_numpy_array_as_pyspark(session):
+    # As PySpark Connect's: a column of each column of a 1- or 2-dimensional 
array, named by the
+    # names, else `value` for one column, else `_1`, `_2`, ..., of the types 
of the array, or
+    # cast to the schema by the server.
+    import numpy as np
+
+    df = session.createDataFrame(np.array([1, 2]))
+    assert df.schema.simpleString() == "struct<value:bigint>"
+    assert sorted(df.collect()) == [Row(value=1), Row(value=2)]
+    df = session.createDataFrame(np.array([[1.5, 2.5], [3.5, 4.5]]))
+    assert df.schema.simpleString() == "struct<_1:double,_2:double>"
+    assert session.createDataFrame(np.array([[1, 2]]), "a string, b 
double").collect() == [
+        Row(a="1", b=2.0)]
+    assert session.createDataFrame(np.array(["x", "y"]), ["s"]).columns == 
["s"]
+    with pytest.raises(PySparkValueError) as e:
+        session.createDataFrame(np.zeros((2, 2, 2)))
+    assert e.value.getCondition() == "INVALID_NDARRAY_DIMENSION"
+    with pytest.raises(PySparkValueError) as e:
+        session.createDataFrame(np.array([[1, 2]]), ["x"])
+    assert e.value.getCondition() == "AXIS_LENGTH_MISMATCH"
+
 def test_create_dataframe_takes_the_ddl_of_spark_as_pyspark(session):
     # As PySpark Connect's, which parses a DDL schema on the server: nested 
structs, arrays and
     # maps of them, backtick-quoted names, and NOT NULL and COMMENT clauses.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b4c5b63..a1aa149 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6043,6 +6043,54 @@ def 
test_create_dataframe_schema_is_names_a_struct_or_an_atomic_type(schema, arg
         "arg_name": "schema", "expected_type": "list, None or StructType", 
"arg_type": arg_type}
 
 
+def test_create_dataframe_of_a_numpy_array_as_pyspark(mock_connect):
+    # As PySpark Connect's: an Arrow table of a column of each column of a 1- 
or 2-dimensional
+    # array, named by the names, else `value` for one column, else `_1`, `_2`, 
..., sent with
+    # the schema, if any, for the server to cast to.
+    import numpy as np
+    import pyarrow as pa
+    from pyspark.errors import PySparkValueError
+
+    spark, service = mock_connect
+
+    def sent(df):
+        df.columns
+        # Relation.local_relation=11: data=1, an Arrow IPC stream, and 
schema=2.
+        relation = _sent_relation(service.analyze_requests[-1], 11)
+        table = pa.ipc.open_stream(relation[1][0]).read_all()
+        return table.to_pydict(), [json.loads(s) for s in relation.get(2, [])]
+
+    assert sent(spark.createDataFrame(np.array([1, 2]))) == ({"value": [1, 
2]}, [])
+    assert sent(spark.createDataFrame(np.array([[1, 2], [3, 4]]))) == (
+        {"_1": [1, 3], "_2": [2, 4]}, [])
+    assert sent(spark.createDataFrame(np.array([[1], [2]]), ("x",))) == ({"x": 
[1, 2]}, [])
+    schema = T.StructType().add("a", T.IntegerType()).add("b", T.StringType())
+    assert sent(spark.createDataFrame(np.array([[1, 2]]), schema)) == (
+        {"_1": [1], "_2": [2]}, [json.loads(schema.json())])
+    assert sent(spark.createDataFrame(np.array([1.5]), T.DoubleType())) == (
+        {"value": [1.5]}, ["double"])
+    for data, schema, expected, actual in [
+        (np.array([1, 2]), ["x", "y"], "2", "1"),
+        (np.array([1, 2]), [], "0", "1"),
+        (np.array([[1, 2]]), ["x"], "1", "2"),
+        (np.array([[1, 2]]), T.IntegerType(), "1", "2"),
+        (np.array([1, 2]), T.StructType().add("a", T.IntegerType()).add("b", 
T.LongType()),
+         "2", "1"),
+    ]:
+        with pytest.raises(PySparkValueError) as e:
+            spark.createDataFrame(data, schema)
+        assert e.value.getCondition() == "AXIS_LENGTH_MISMATCH"
+        assert e.value.getMessageParameters() == {
+            "expected_length": expected, "actual_length": actual}
+    for data in (np.zeros((2, 2, 2)), np.array(5)):
+        with pytest.raises(PySparkValueError) as e:
+            spark.createDataFrame(data)
+        assert e.value.getCondition() == "INVALID_NDARRAY_DIMENSION"
+        assert e.value.getMessageParameters() == {"dimensions": "1 or 2"}
+    with pytest.raises(PySparkValueError) as e:
+        spark.createDataFrame(np.array([]))
+    assert e.value.getCondition() == "CANNOT_INFER_EMPTY_SCHEMA"
+
 def test_create_dataframe_of_a_dataframe_raises_as_pyspark():
     from pyspark.errors import PySparkTypeError
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to