This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new e22a657  [SPARK-61079][CONNECT][PYTHON] Convert pandas input with an 
explicit schema as PySpark's Arrow conversion does
e22a657 is described below

commit e22a657e16b0cd46dad6a102ea2b9228ffe6f20a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:50:23 2026 +0900

    [SPARK-61079][CONNECT][PYTHON] Convert pandas input with an explicit schema 
as PySpark's Arrow conversion does
    
    ### What changes were proposed in this pull request?
    
    `createDataFrame` of a pandas DataFrame with an explicit schema (a 
`StructType` or a DDL string) now converts it as PySpark 4.3.0 does: with the 
vendored official `create_arrow_table_from_pandas` (`pyspark.sql.conversion`, 
unchanged), after reading the Arrow options, the session time zone and 
`spark.sql.execution.pandas.convertToArrowArraySafely` as PySpark does, and 
sends the table through the Arrow importer, which uploads it at the relation 
cache threshold (`create_from_pandas_arrow [...]
    
    ### Why are the changes needed?
    
    [SPARK-61079](https://issues.apache.org/jira/browse/SPARK-61079): the row 
conversion of pandas cells differed from PySpark's Arrow conversion. Against a 
local Spark 4.3.0 server:
    
    | Input | Current master | This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `{"f": [1, None]}` or `[1.0, nan]` into `f int` | `PySparkRuntimeError: 
Type mismatch in row data` | `[Row(f=1), Row(f=None)]` |
    | int64 `257` into `b tinyint` | `[Row(b=1)]`, silently | 
`PySparkValueError` (Arrow's overflow) |
    | a null in a non-nullable field | `PySparkRuntimeError: Failed to create 
Arrow batch` | `ValueError: Casting field 'a' with null values to non-nullable` 
|
    | naive `2020-07-01 12:00`, client `TZ=America/Los_Angeles`, session UTC | 
collects as 12:00 | collects as 05:00 (stored as 12:00 UTC); a DST gap raises 
as PySpark |
    | a timestamp with nanoseconds | truncated silently | `PySparkValueError` |
    | a lone surrogate into `string` | `TypeError` | `PySparkValueError` |
    | a `datetime` into `date` | `PySparkRuntimeError` | `date(2020, 1, 2)` |
    | NaN nested in arrays and maps | kept as NaN | `None` |
    | `2**64` in an inferred object column | a double | `OverflowError` |
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these inputs convert, or fail, as in PySpark. Explicit-schema pandas 
input reads its configs as PySpark does before converting.
    
    ### How was this patch tested?
    
    - 30 new tests in `python/tests/test_arrow_handoff.py`, against the mock 
Arrow input service, for each case above, the time-zone variants and the DST 
gaps; existing tests that pinned the row path's errors or its config order now 
expect PySpark's (listed in the commits).
    - `test_arrow_handoff.py` (903 passed) and the pandas and createDataFrame 
tests of the offline suite; each case above and both time-zone variants give 
the same output as PySpark 4.3.0.
    
    Still different: a frame whose width differs from the schema raises 
`AXIS_LENGTH_MISMATCH`, where PySpark drops a column or raises `ArrowInvalid`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #270 from HyukjinKwon/SPARK-61079-pandas-explicit-schema.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs     |  95 +++++++++++-
 crates/spark-connect/src/session.rs  |   7 +-
 python/pyspark/sql/_pandas_input.py  | 155 +++++++++++++++++++
 python/tests/_mock_connect_server.py |  10 ++
 python/tests/input_name_cases.py     |   9 ++
 python/tests/integer_tuple_cases.py  |  15 +-
 python/tests/test_arrow_handoff.py   | 279 +++++++++++++++++++++++++++++++----
 python/tests/test_dropin_offline.py  |  42 ++++--
 8 files changed, 566 insertions(+), 46 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index c6dc346..c62377a 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -544,6 +544,77 @@ impl PySparkSession {
         }))
     }
 
+    /// A DataFrame of a non-empty pandas DataFrame under an explicit struct 
schema of the
+    /// same width, converted as PySpark does: each column by its Arrow 
conversion (session
+    /// time zone, safe casts), then the table cast to the schema, which the 
importer
+    /// sends as Arrow. None when the width differs, which the row path 
reports.
+    fn create_from_pandas_arrow(
+        &self,
+        py: Python<'_>,
+        data: &Bound<'_, PyAny>,
+        schema: &Bound<'_, PyAny>,
+        fields: &[StructField],
+    ) -> PyResult<Option<PyDataFrame>> {
+        if fields.is_empty() || data.len()? == 0 || 
data.getattr("columns")?.len()? != fields.len()
+        {
+            return Ok(None);
+        }
+        // The caller's schema object is checked, before any configuration 
request is sent.
+        let helper = py.import("pyspark.sql._pandas_input")?;
+        if !helper
+            .getattr("accepts_pandas_table")?
+            .call1((data, schema))?
+            .is_truthy()?
+        {
+            return Ok(None);
+        }
+        let options = py.detach(|| 
self.session.arrow_input_options().to_pyerr())?;
+        // The option is always set, as the importer requires it.
+        let timezone = options.timezone().unwrap_or("UTC").to_owned();
+        let large = options.use_large_var_types() == Some("true");
+        let safecheck = py
+            .detach(|| {
+                self.session
+                    .conf()
+                    
.get("spark.sql.execution.pandas.convertToArrowArraySafely")
+            })
+            .to_pyerr()?
+            .as_deref()
+            == Some("true");
+        // The caller's StructType, which the guard checked, is converted as 
given; a DDL
+        // string is parsed into the same fields.
+        let schema_py = if schema
+            .extract::<PyRef<crate::types::PyStructType>>()
+            .is_ok()
+        {
+            schema.clone().unbind()
+        } else {
+            crate::types::data_type_to_py(
+                py,
+                &DataType::Struct {
+                    fields: fields.to_vec(),
+                },
+            )?
+        };
+        let table = helper.getattr("prepare_pandas_table")?.call1((
+            data,
+            schema_py.bind(py),
+            timezone,
+            safecheck,
+            large,
+        ))?;
+        let frame = crate::pyarrow_import::create_dataframe(
+            py,
+            &self.session,
+            &table,
+            Some(schema_py.bind(py)),
+        )?;
+        // As the other pandas paths, the explicit names are applied with toDF.
+        Ok(Some(PyDataFrame::new(frame.dataframe.to_df(
+            fields.iter().map(|field| field.name.as_str()).collect(),
+        ))))
+    }
+
     /// A DataFrame of a pandas DataFrame: its rows as lists with NaN and NaT 
as None, typed
     /// by the first non-null value of each column, and its column labels, as 
strings, or the
     /// given names, applied with `toDF`.
@@ -553,6 +624,9 @@ impl PySparkSession {
         data: &Bound<'_, PyAny>,
         spec: Spec,
     ) -> PyResult<PyDataFrame> {
+        // Without a struct schema, PySpark infers the Arrow type of each 
column, which has
+        // no int64 for a Python int beyond the 64-bit range.
+        let inferred = !matches!(spec, Spec::Struct(_));
         let mut columns = Vec::new();
         for label in data.getattr("columns")?.try_iter()? {
             columns.push(label?.str()?.extract::<String>()?);
@@ -614,7 +688,19 @@ impl PySparkSession {
         for row in data.iter() {
             let values = row
                 .try_iter()?
-                .map(|v| py_to_value(&v?))
+                .map(|v| {
+                    let v = v?;
+                    if inferred {
+                        if let Ok(i) = v.cast_exact::<pyo3::types::PyInt>() {
+                            if i.extract::<i64>().is_err() {
+                                return 
Err(pyo3::exceptions::PyOverflowError::new_err(
+                                    "Python int too large to convert to C 
long",
+                                ));
+                            }
+                        }
+                    }
+                    py_to_value(&v)
+                })
                 .collect::<PyResult<Vec<_>>>()?;
             if values.len() != row_names.len() {
                 // As pyspark, which checks every row against the width of the 
schema.
@@ -1159,6 +1245,13 @@ impl PySparkSession {
                     _ => frame,
                 });
             }
+            // An explicit struct schema converts with PySpark's Arrow 
conversion, which
+            // reports the same values and errors. Other schemas keep the row 
path.
+            if let (Spec::Struct(fields), Some(schema)) = (&spec, schema) {
+                if let Some(frame) = self.create_from_pandas_arrow(py, data, 
schema, fields)? {
+                    return Ok(frame);
+                }
+            }
             return self.create_from_pandas(py, data, spec);
         }
         // Preserve schema parsing, ignored-argument warnings, and 
pandas/Arrow routing.
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 8e5201b..0b57655 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -986,9 +986,10 @@ fn row_width_mismatch(row_index: usize, actual: usize, 
expected: usize) -> Spark
 /// Convert Arrow RecordBatch data to IPC bytes.
 /// Used by createDataFrame to package local data.
 /// The size of a local relation, in Arrow data, from which `createDataFrame` 
reads the
-/// server's local-relation settings to decide whether to upload it: half of 
the default
-/// `spark.sql.session.localRelationCacheThreshold`, and well below the 
default gRPC message
-/// size limit of the server (128 MiB). Smaller relations are sent inline 
without that RPC.
+/// server's local-relation settings to decide whether to upload it, well 
below the default
+/// gRPC message size limit of the server (128 MiB). Smaller relations are 
sent inline without
+/// that RPC, even above `spark.sql.session.localRelationCacheThreshold` (1 
MiB by default),
+/// at which PySpark uploads them.
 const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20;
 
 fn rows_to_arrow_batch(
diff --git a/python/pyspark/sql/_pandas_input.py 
b/python/pyspark/sql/_pandas_input.py
index 2797219..f4891c2 100644
--- a/python/pyspark/sql/_pandas_input.py
+++ b/python/pyspark/sql/_pandas_input.py
@@ -813,6 +813,161 @@ def prepare_date_table(frame, schema):
     )
 
 
+def _native_names_intact(namespace):
+    """Whether each public native class name in `namespace` is still the 
native class."""
+    from pyspark import _pyspark as native
+
+    return all(
+        namespace[name] is value
+        for name, value in vars(native).items()
+        if not name.startswith("_") and isinstance(value, type) and name in 
namespace
+    )
+
+
+def _canonical_struct(schema):
+    """Whether a native struct schema and its fields keep their own classes' 
behavior.
+
+    The conversion reads the fields, names and data types through the classes, 
so a
+    customized class keeps the row path.
+    """
+    from types import GetSetDescriptorType, WrapperDescriptorType
+
+    from pyspark._pyspark import StructField, StructType
+
+    if type(schema) is not StructType:
+        return False
+    for cls in (StructType, StructField):
+        if (cls.__getattribute__ is not object.__getattribute__
+                or any("__class__" in vars(base) for base in cls.__mro__ if 
base is not object)):
+            return False
+    for cls, names, descriptor_type in (
+        (StructType, ("fields", "names"), GetSetDescriptorType),
+        (StructType, ("__iter__",), WrapperDescriptorType),
+        (StructField, ("name", "dataType", "nullable", "metadata"), 
GetSetDescriptorType),
+    ):
+        for name in names:
+            descriptor = vars(cls).get(name)
+            if (type(descriptor) is not descriptor_type
+                    or descriptor.__objclass__ is not cls or 
descriptor.__name__ != name):
+                return False
+    return all(
+        type(field) is StructField and type(field.name) is str and "\x00" not 
in field.name
+        and type(field.dataType).__getattribute__ is object.__getattribute__
+        for field in schema.fields
+    )
+
+
+def accepts_pandas_table(frame, schema):
+    """Whether `prepare_pandas_table` may convert this frame under this schema.
+
+    Customized frames, columns, schema classes and rebound type names keep the 
row path,
+    which reads them in its own order. The conversion and the importer resolve 
the type
+    names by lookup, so each namespace must still hold the native classes. The 
schema is
+    the caller's object, or a DDL string, which is parsed.
+    """
+    import sys
+
+    import pandas as pd
+    from pandas.core.frame import DataFrame
+
+    from pyspark.sql import types as sql_types
+
+    if pd.DataFrame is not DataFrame or type(frame) is not DataFrame:
+        return False
+    if not _has_canonical_columns(frame.columns):
+        return False
+    # Custom extension arrays and dtypes keep the row path: Arrow would call 
their own
+    # `__arrow_array__`, and their dtype may impersonate a NumPy type.
+    for i in range(len(frame.columns)):
+        column = frame.iloc[:, i]
+        if not all(type(value).__module__.split(".")[0] in ("pandas", "numpy")
+                   for value in (column.array, column.dtype)):
+            return False
+    # The native row path normalizes cells through these methods, which a 
customized class
+    # or instance keeps on the row path.
+    from pandas.core import frame as frame_module, generic as generic_module
+    from types import FunctionType
+
+    if any(name in vars(frame) for name in ("astype", "notna", "where")):
+        return False
+    for name, module, qualname in (
+        ("astype", generic_module, "NDFrame.astype"),
+        ("notna", frame_module, "DataFrame.notna"),
+        ("where", generic_module, "NDFrame.where"),
+    ):
+        method = getattr(DataFrame, name)
+        if (type(method) is not FunctionType or method.__qualname__ != qualname
+                or method.__module__ != module.__name__ or method.__globals__ 
is not vars(module)):
+            return False
+    if sys.modules.get("pyspark.sql.types") is not sql_types or not 
_native_names_intact(
+        vars(sql_types)
+    ):
+        return False
+    for name in ("pyspark.sql.pandas.types", "pyspark.sql._arrow_input"):
+        module = sys.modules.get(name)
+        if module is not None and not _native_names_intact(vars(module)):
+            return False
+    # The importer resolves its schema classes by name, and its conversion 
functions may
+    # keep the globals of an older type module.
+    importer = sys.modules.get("pyspark.sql._arrow_input")
+    if importer is not None:
+        from pyspark._pyspark import AtomicType, StructType
+
+        if any(vars(importer).get(name) is not cls
+               for name, cls in (("StructType", StructType), ("AtomicType", 
AtomicType))):
+            return False
+        for function in list(vars(importer).values()):
+            namespace = getattr(function, "__globals__", None)
+            if type(namespace) is dict and not _native_names_intact(namespace):
+                return False
+        # The importer calls its converters by these names, which must still 
be functions.
+        converter = sys.modules.get("pyspark.sql.pandas.types")
+        for name in ("to_arrow_schema", "from_arrow_schema"):
+            if name in vars(importer) and (
+                getattr(vars(importer)[name], "__globals__", None) is None
+                or (converter is not None
+                    and vars(importer)[name] is not getattr(converter, name, 
None))
+            ):
+                return False
+    return type(schema) is str or _canonical_struct(schema)
+
+
+def prepare_pandas_table(frame, schema, timezone, safecheck, 
prefers_large_types):
+    """Convert a pandas frame under an explicit struct schema as PySpark 
Connect does.
+
+    Each column goes through PySpark's own Arrow conversion, with the session 
time zone
+    and the safe-cast setting, and the table is then renamed and cast to the 
schema's
+    Arrow types. The values and errors therefore match PySpark, including the 
nullability
+    and overflow messages. The caller checks the frame with 
`accepts_pandas_table` and
+    the column count against the schema first.
+    """
+    from pyspark.sql.pandas.conversion import create_arrow_table_from_pandas
+    from pyspark.sql.pandas.types import _deduplicate_field_names, 
to_arrow_schema
+
+    import pyarrow as pa
+
+    deduplicated = _deduplicate_field_names(schema)
+    arrow_schema = to_arrow_schema(
+        deduplicated, timezone="UTC", prefers_large_types=prefers_large_types
+    )
+    # Top-level names are the caller's, which may repeat: the importer casts 
to them.
+    arrow_schema = pa.schema([
+        pa.field(name, field.type, field.nullable, field.metadata)
+        for name, field in zip(schema.names, arrow_schema)
+    ])
+    series_with_types = [
+        (column, field.dataType)
+        for (_, column), field in zip(frame.items(), deduplicated.fields)
+    ]
+    table = create_arrow_table_from_pandas(
+        series_with_types,
+        timezone=timezone,
+        safecheck=safecheck,
+        prefers_large_types=prefers_large_types,
+    )
+    return table.rename_columns(schema.names).cast(arrow_schema)
+
+
 def arrow_decimal_column(values, dtype, name):
     """The precision, scale and values of a pandas object column of Decimals 
and other values,
     as Arrow types and converts it: PySpark's conversion of pandas leaves such 
a column to
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index 6822888..1699501 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -342,6 +342,16 @@ def serve(connection):
         default_confs = {
             "spark.sql.execution.pythonUDF.arrow.enabled": "true",
             "spark.sql.execution.pythonUDTF.arrow.enabled": "true",
+            # Those that the Arrow conversion of pandas input reads, as a 
Spark 4.3.0 server.
+            "spark.sql.session.timeZone": "UTC",
+            "spark.sql.timestampType": "TIMESTAMP_LTZ",
+            "spark.sql.execution.arrow.useLargeVarTypes": "false",
+            "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
+            "spark.sql.session.localRelationCacheThreshold": "1048576",
+            "spark.sql.session.localRelationSizeLimit": "3221225472",
+            "spark.sql.session.localRelationChunkSizeRows": "10000",
+            "spark.sql.session.localRelationChunkSizeBytes": "16777216",
+            "spark.sql.session.localRelationBatchOfChunksSizeBytes": 
"1073741824",
         }
         read_confs = []
         set_confs = []
diff --git a/python/tests/input_name_cases.py b/python/tests/input_name_cases.py
index 4bc5f43..6462b00 100644
--- a/python/tests/input_name_cases.py
+++ b/python/tests/input_name_cases.py
@@ -103,6 +103,15 @@ class InputNameTests(unittest.TestCase):
                 b"spark.sql.session.localRelationCacheThreshold": str(1 << 
60).encode(),
                 b"spark.sql.connect.serverStacktrace.enabled": b"false",
                 b"spark.sql.pyspark.jvmStacktrace.enabled": b"false",
+                # Explicit-schema pandas input reads the Arrow conversion 
settings first.
+                b"spark.sql.timestampType": b"TIMESTAMP_LTZ",
+                b"spark.sql.session.timeZone": b"UTC",
+                b"spark.sql.execution.arrow.useLargeVarTypes": b"false",
+                b"spark.sql.session.localRelationSizeLimit": str(1 << 
60).encode(),
+                b"spark.sql.session.localRelationChunkSizeRows": str(1 << 
30).encode(),
+                b"spark.sql.session.localRelationChunkSizeBytes": str(1 << 
40).encode(),
+                b"spark.sql.session.localRelationBatchOfChunksSizeBytes": 
str(1 << 40).encode(),
+                b"spark.sql.execution.pandas.convertToArrowArraySafely": 
b"true",
             }
             pairs = b"".join(_bytes(2, _bytes(1, key) + _bytes(2, values[key]))
                              for key in keys)
diff --git a/python/tests/integer_tuple_cases.py 
b/python/tests/integer_tuple_cases.py
index 184d4dc..2a6bd3f 100644
--- a/python/tests/integer_tuple_cases.py
+++ b/python/tests/integer_tuple_cases.py
@@ -102,6 +102,15 @@ class IntegerTupleTests(unittest.TestCase):
                 b"spark.sql.session.localRelationCacheThreshold": str(1 << 
60).encode(),
                 b"spark.sql.connect.serverStacktrace.enabled": b"false",
                 b"spark.sql.pyspark.jvmStacktrace.enabled": b"false",
+                # Explicit-schema pandas input reads the Arrow conversion 
settings first.
+                b"spark.sql.timestampType": b"TIMESTAMP_LTZ",
+                b"spark.sql.session.timeZone": b"UTC",
+                b"spark.sql.execution.arrow.useLargeVarTypes": b"false",
+                b"spark.sql.session.localRelationSizeLimit": str(1 << 
60).encode(),
+                b"spark.sql.session.localRelationChunkSizeRows": str(1 << 
30).encode(),
+                b"spark.sql.session.localRelationChunkSizeBytes": str(1 << 
40).encode(),
+                b"spark.sql.session.localRelationBatchOfChunksSizeBytes": 
str(1 << 40).encode(),
+                b"spark.sql.execution.pandas.convertToArrowArraySafely": 
b"true",
             }
             pairs = b"".join(_bytes(2, _bytes(1, key) + _bytes(2, values[key]))
                              for key in keys)
@@ -134,7 +143,11 @@ class IntegerTupleTests(unittest.TestCase):
         config_before = self.config_calls
         frame = self.spark.createDataFrame(data, schema)
         self.assertEqual(len(self.captured), before)  # construction remains 
offline
-        self.assertEqual(self.config_calls, config_before)
+        # Explicit-schema pandas input reads the Arrow conversion settings 
during
+        # construction, as PySpark does: the normalization and safe-cast 
configuration.
+        expected = 3 if isinstance(data, pd.DataFrame) else 0
+        self.assertEqual(self.config_calls - config_before, expected)
+        config_before = self.config_calls
         frame.toArrow()
         self.assertEqual(len(self.captured), before + 1)
         self.assertEqual(self.config_calls, config_before)
diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index 2f9f58b..b40a434 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -451,6 +451,7 @@ def _arrow_input_service(
         "spark.sql.timestampType": "TIMESTAMP_LTZ",
         "spark.sql.session.timeZone": "UTC",
         "spark.sql.execution.arrow.useLargeVarTypes": "false",
+        "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
         "spark.sql.session.localRelationCacheThreshold": "1048576",
         "spark.sql.session.localRelationSizeLimit": str(3 * 1024**3),
         "spark.sql.session.localRelationChunkSizeRows": "10000",
@@ -1555,12 +1556,19 @@ def test_non_nullable_schema_is_preserved(protocol, 
verify, config_error):
     frame = pd.DataFrame({"a": pd.Series([1, None], dtype="Int64")})
     schema = T.StructType([T.StructField("a", T.LongType(), False)])
     assert prepare_int64_table(frame, schema) is None
-    with _arrow_input_service(protocol, config_error=config_error) as (spark, 
state):
-        with pytest.raises(RuntimeError, match="null"):
-            spark.createDataFrame(frame, schema, verifySchema=verify)
+    with _arrow_input_service(protocol) as (spark, state):
+        if config_error:
+            state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+            with pytest.raises(Exception, match="configuration read failed"):
+                spark.createDataFrame(frame, schema, verifySchema=verify)
+        else:
+            # Official reads the configuration first, then the cast fails with 
a ValueError.
+            with pytest.raises(ValueError) as error:
+                spark.createDataFrame(frame, schema, verifySchema=verify)
+            assert str(error.value) == "Casting field 'a' with null values to 
non-nullable"
+        assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)
         assert not state["relations"]
         assert not state["uploads"]
-        assert state["config_requests"] == []
 
 
 def test_small_numpy_frames_avoid_importer_configuration_overhead():
@@ -1622,7 +1630,7 @@ def 
test_integer_explicit_renamed_schema_preserves_all_ipc_cells(protocol, monke
                 from_arrow_schema(table.schema).json(), schema.json(), 
table.schema)
             assert state["config_requests"]
         else:
-            assert state["config_requests"] == []
+            assert state["config_requests"][:1] == [("get", 
_NORMALIZATION_KEYS + _UPLOAD_KEYS)]
 
 
 def test_integer_dataframe_and_name_subclasses_keep_fallback(monkeypatch):
@@ -2162,8 +2170,10 @@ def test_pandas_timestamp_non_nullable_schema(protocol, 
has_null):
     schema = T.StructType([T.StructField("a", T.TimestampType(), False)])
     with _arrow_input_service(protocol, capture_tables=True) as (spark, state):
         if has_null:
-            with pytest.raises(RuntimeError, match="null"):
+            # Official: the cast of the Arrow table, a ValueError rather than 
a row error.
+            with pytest.raises(ValueError) as error:
                 spark.createDataFrame(frame, schema)
+            assert str(error.value) == "Casting field 'a' with null values to 
non-nullable"
             assert not state["relations"]
         else:
             spark.createDataFrame(frame, schema).toArrow()
@@ -2184,17 +2194,25 @@ def 
test_pandas_timestamp_mixed_and_arrow_dtypes_fallback():
 
 @pytest.mark.parametrize("config_error", [False, True])
 @pytest.mark.parametrize("verify", [False, True])
-def test_timestamp_nonnullable_nat_keeps_local_error_before_config(protocol, 
config_error, verify):
+def test_timestamp_nonnullable_nat_reads_config_before_cast_error(protocol, 
config_error, verify):
     from pyspark.sql import types as T
     from pyspark.sql._pandas_input import timestamp_columns
 
     frame = pd.DataFrame({"a": pd.to_datetime([0, None], utc=True)})
     schema = T.StructType([T.StructField("renamed", T.TimestampType(), False)])
     assert timestamp_columns(frame, schema) is None
-    with _arrow_input_service(protocol, capture_tables=True, 
config_error=config_error) as (spark, state):
-        with pytest.raises(RuntimeError, match="null"):
-            spark.createDataFrame(frame, schema, verifySchema=verify)
-        assert state["config_requests"] == []
+    with _arrow_input_service(protocol, capture_tables=True) as (spark, state):
+        if config_error:
+            state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+            with pytest.raises(Exception, match="configuration read failed"):
+                spark.createDataFrame(frame, schema, verifySchema=verify)
+        else:
+            # Official: ValueError from the cast of the Arrow table to the 
schema.
+            with pytest.raises(ValueError) as error:
+                spark.createDataFrame(frame, schema, verifySchema=verify)
+            assert str(error.value) == (
+                "Casting field 'renamed' with null values to non-nullable")
+        assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)
         assert state["relations"] == []
         assert state["uploads"] == 0
 
@@ -2693,7 +2711,11 @@ def _assert_float64_ipc(table, columns, schema, *, 
missing=None):
         np.testing.assert_array_equal(actual.view(np.uint64), 
expected[~mask].view(np.uint64))
         actual_field = table.schema.field(index)
         assert actual_field.nullable == field.nullable
-        assert not actual_field.metadata
+        # The Arrow conversion keeps the field metadata; the fast path's table 
has none.
+        from pyspark.sql.pandas.types import from_arrow_metadata
+
+        if actual_field.metadata is not None:
+            assert from_arrow_metadata(actual_field.metadata) == field.metadata
 
 
 
@@ -2755,8 +2777,10 @@ def 
test_float64_columnar_ipc_matches_existing_row_conversion(protocol, monkeypa
         assert len(state["input_tables"]) == 2
         for table in state["input_tables"]:
             _assert_float64_ipc(table, [values], schema)
-        assert state["input_tables"][0].equals(state["input_tables"][1], 
check_metadata=True)
-        assert state["config_requests"] == []
+        # The Arrow conversion of the second call carries the field metadata 
in its table
+        # and reads the configuration first, as official; the values and types 
are the same.
+        assert state["input_tables"][0].equals(state["input_tables"][1], 
check_metadata=False)
+        assert state["config_requests"][:1] == [("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)]
         assert state["uploads"] == 0
         assert [json.loads(relation.local_relation.schema) for relation in 
state["relations"]] == [
             json.loads(schema.json()), json.loads(schema.json())
@@ -2778,11 +2802,12 @@ def 
test_float64_pandas_infinity_missing_option_keeps_row_semantics(protocol, nu
     with pd.option_context("mode.use_inf_as_na", True):
         with _arrow_input_service(protocol) as (spark, state):
             if not nullable:
-                with pytest.raises(PySparkRuntimeError, match=(
-                    "Column 'x' is declared as non-nullable but contains null 
values"
-                )):
+                # Official: the configuration is read, then the cast fails 
with a ValueError.
+                with pytest.raises(ValueError) as error:
                     spark.createDataFrame(frame, schema)
-                assert state["config_requests"] == []
+                assert str(error.value) == "Casting field 'x' with null values 
to non-nullable"
+                assert state["config_requests"][:1] == [
+                    ("get", _NORMALIZATION_KEYS + _UPLOAD_KEYS)]
                 assert state["uploads"] == 0
                 return
             spark.createDataFrame(frame, schema).toArrow()
@@ -2901,26 +2926,31 @@ def 
test_float64_public_fallback_preserves_every_ipc_cell(protocol, reason):
         schema = expected_schema = T.StructType([T.StructField("x\x00renamed", 
T.DoubleType())])
     with _arrow_input_service(protocol) as (spark, state):
         spark.createDataFrame(frame, schema).toArrow()
-        assert state["config_requests"] == []
+        # Official converts explicit schemas with Arrow, reading the 
configuration first.
+        native = reason in ("inferred", "names", "frame_subclass", 
"schema_nul")
+        assert state["config_requests"][:1] == (
+            [] if native else [("get", _NORMALIZATION_KEYS + _UPLOAD_KEYS)])
         _assert_float64_ipc(state["input_tables"][0], columns, expected_schema)
 
 
 
 @pytest.mark.parametrize("verify", [False, True])
-def test_float64_nonnullable_nan_keeps_existing_error_before_configs(protocol, 
verify):
-    from pyspark.errors import PySparkRuntimeError
-
+def test_float64_nonnullable_nan_reads_config_before_cast_error(protocol, 
verify):
     frame = pd.DataFrame({"x": np.full(100000, np.nan, dtype=np.float64)})
     schema = T.StructType([T.StructField("x", T.DoubleType(), False)])
     with _arrow_input_service(protocol) as (spark, state):
         state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
-        with pytest.raises(PySparkRuntimeError, match=(
-            "Column 'x' is declared as non-nullable but contains null values"
-        )):
+        # Official reads the configuration first, so its error precedes the 
cast.
+        with pytest.raises(Exception, match="configuration read failed"):
             spark.createDataFrame(frame, schema, verifySchema=verify)
-        assert state["config_requests"] == []
+        assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)
         assert state["relations"] == []
         assert state["uploads"] == 0
+        # Then the cast fails as official does, with a ValueError.
+        with pytest.raises(ValueError) as error:
+            spark.createDataFrame(frame, schema, verifySchema=verify)
+        assert str(error.value) == "Casting field 'x' with null values to 
non-nullable"
+        assert state["relations"] == []
 
 
 
@@ -3303,7 +3333,7 @@ def 
test_float64_missing_concrete_numpy_backing_class_keeps_fallback(monkeypatch
 
 
 @pytest.mark.parametrize("rows", [10000, 99999, 100000])
-def test_float64_public_threshold_preserves_row_path_below_100k(protocol, 
rows):
+def test_float64_public_threshold_reads_config_below_100k(protocol, rows):
     import json
 
     values = np.resize(np.array([np.nan, -0.0, 0.0, np.inf, -np.inf, 1.5]), 
rows)
@@ -3314,12 +3344,21 @@ def 
test_float64_public_threshold_preserves_row_path_below_100k(protocol, rows):
     with _arrow_input_service(
         protocol, **{"spark.sql.session.localRelationCacheThreshold": 
"999999999"}
     ) as (spark, state):
+        if rows >= 100000:
+            # The float64 fast path converts at this size without a 
configuration read.
+            spark.createDataFrame(frame, schema).toArrow()
+            _assert_float64_ipc(state["input_tables"][0], [values], schema)
+            assert state["config_requests"] == []
+            return
         state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+        # Official reads the configuration before converting, so its error 
comes first.
+        with pytest.raises(Exception, match="configuration read failed"):
+            spark.createDataFrame(frame, schema)
         spark.createDataFrame(frame, schema).toArrow()
         _assert_float64_ipc(state["input_tables"][0], [values], schema)
         assert json.loads(state["relations"][0].local_relation.schema) == 
json.loads(schema.json())
         assert state["uploads"] == 0
-        assert state["config_requests"] == []
+        assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)
 
 
 
@@ -4198,7 +4237,6 @@ def 
test_string_public_fallback_keeps_row_semantics(protocol, monkeypatch, reaso
         assert state["config_requests"] == []
 
 
-
 @pytest.mark.parametrize("surrogate", ["\ud800", "\udfff"])
 def test_string_surrogate_fallback_precedes_importer_configuration(protocol, 
monkeypatch, surrogate):
     values = ["plain"] * 10000
@@ -4223,7 +4261,6 @@ def 
test_string_surrogate_fallback_precedes_importer_configuration(protocol, mon
         assert state["uploads"] == 0
 
 
-
 @pytest.mark.parametrize("class_name,attribute", [
     ("StructType", "names"), ("StructType", "fields"), ("StructType", 
"__iter__"),
     ("StructField", "name"), ("StructField", "dataType"), ("StructField", 
"nullable"),
@@ -4284,7 +4321,12 @@ class Factory:
         return Path(tempfile.mkdtemp(prefix=name))
 with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state):
     spark.createDataFrame(frame, schema).toArrow()
-    assert state["config_requests"] == [], state["config_requests"]
+    if attribute == "isUTF8BinaryCollation":
+        # Official converts this explicit schema with Arrow, after reading the 
configuration;
+        # it never reads this attribute, so the result is the same as the row 
path's.
+        assert state["config_requests"][0][0] == "get", 
state["config_requests"]
+    else:
+        assert state["config_requests"] == [], state["config_requests"]
     table = state["input_tables"][0]
     assert table.schema.names == ["text"], table.schema
     assert table.column(0).to_pylist() == ["plain"] * 10000
@@ -4543,6 +4585,181 @@ def 
test_string_large_inline_input_keeps_row_transport_and_schema(protocol):
             __import__("json").loads(schema.json()))
 
 
+# Explicit-schema pandas input converts as PySpark's Arrow conversion does: 
the session time
+# zone, the safe-cast setting, the nullability and the errors. Each 
expectation below was
+# checked against official PySpark 4.3.0 with the same frame, schema and 
session settings.
+
+
+def _pandas_table(protocol, frame, schema, **overrides):
+    """The table that an explicit-schema pandas frame sends, and whether it is 
local."""
+    with _arrow_input_service(protocol, **overrides) as (spark, state):
+        df = spark.createDataFrame(frame, schema)
+        assert not df.isLocal()
+        df.toArrow()
+        return state["input_tables"][0]
+
+
+def _in_host_zone(name, call):
+    """Run `call` with the process time zone `name`, as the client's local 
time zone."""
+    import time
+
+    previous = os.environ.get("TZ")
+    os.environ["TZ"] = name
+    time.tzset()
+    try:
+        return call()
+    finally:
+        if previous is None:
+            os.environ.pop("TZ", None)
+        else:
+            os.environ["TZ"] = previous
+        time.tzset()
+
+
[email protected]("values", [[1, None], [1.0, np.nan]], ids=["int", 
"float"])
+def test_pandas_explicit_int_nulls_are_null(protocol, values):
+    table = _pandas_table(protocol, pd.DataFrame({"f": values}), "f int")
+    # Official: [Row(f=1), Row(f=None)].
+    assert table.schema.field("f").type == pa.int32()
+    assert table.column("f").to_pylist() == [1, None]
+
+
+def _tinyint_schema():
+    return T.StructType([T.StructField("b", T.ByteType())])
+
+
+def test_pandas_explicit_tinyint_overflow_raises_as_pyspark(protocol):
+    from pyspark.errors import PySparkValueError
+
+    with _arrow_input_service(protocol) as (spark, state):
+        with pytest.raises(PySparkValueError) as error:
+            spark.createDataFrame(pd.DataFrame({"b": [257]}), 
_tinyint_schema())
+    assert str(error.value) == (
+        "Exception thrown when converting pandas.Series (int64) with name 'b' 
to Arrow Array "
+        "(int8). It can be caused by overflows or other unsafe conversions 
warned by Arrow. "
+        "Arrow safe type check can be disabled by using SQL config "
+        "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+    )
+    assert state["input_tables"] == []
+
+
+def test_pandas_explicit_tinyint_overflow_wraps_when_unsafe(protocol):
+    # Official, with convertToArrowArraySafely=false: [Row(b=1)].
+    table = _pandas_table(
+        protocol, pd.DataFrame({"b": [257]}), _tinyint_schema(),
+        **{"spark.sql.execution.pandas.convertToArrowArraySafely": "false"},
+    )
+    assert table.column("b").to_pylist() == [1]
+
+
+def test_pandas_explicit_nonnullable_null_raises_as_pyspark(protocol):
+    schema = T.StructType([T.StructField("a", T.LongType(), False)])
+    frame = pd.DataFrame({"a": pd.array([1, None], dtype="Int64")})
+    with _arrow_input_service(protocol) as (spark, state):
+        with pytest.raises(ValueError) as error:
+            spark.createDataFrame(frame, schema)
+    # Official: ValueError, not a PySparkRuntimeError from the row path.
+    assert type(error.value) is ValueError
+    assert str(error.value) == "Casting field 'a' with null values to 
non-nullable"
+    assert state["input_tables"] == []
+
+
[email protected]("host", ["America/Los_Angeles", "UTC"])
+def test_pandas_naive_timestamp_uses_session_zone(protocol, host):
+    frame = pd.DataFrame({"t": [pd.Timestamp("2020-07-01 12:00")]})
+    # Official with session UTC, whatever the client zone: 12:00 UTC.
+    table = _in_host_zone(
+        host,
+        lambda: _pandas_table(
+            protocol, frame, "t timestamp",
+            **{"spark.sql.session.timeZone": "UTC"},
+        ),
+    )
+    assert table.column("t").to_pylist() == [datetime(2020, 7, 1, 12, 
tzinfo=timezone.utc)]
+
+
+def test_pandas_naive_timestamp_in_session_zone_is_converted(protocol):
+    pytz = pytest.importorskip("pytz")
+    frame = pd.DataFrame({"t": [pd.Timestamp("2020-07-01 12:00")]})
+    # Official with session America/Los_Angeles and a client zone of UTC: 
19:00 UTC.
+    table = _in_host_zone(
+        "UTC",
+        lambda: _pandas_table(
+            protocol, frame, "t timestamp",
+            **{"spark.sql.session.timeZone": "America/Los_Angeles"},
+        ),
+    )
+    assert table.column("t").to_pylist() == [datetime(2020, 7, 1, 19, 
tzinfo=timezone.utc)]
+
+    gap = pd.DataFrame({"t": [pd.Timestamp("2020-03-08 02:30")]})
+    with _arrow_input_service(protocol, **{
+        "spark.sql.session.timeZone": "America/Los_Angeles",
+    }) as (spark, _):
+        # Official: pytz NonExistentTimeError for a naive time inside the DST 
gap.
+        with pytest.raises(pytz.exceptions.NonExistentTimeError):
+            spark.createDataFrame(gap, "t timestamp")
+
+
+def test_pandas_nanosecond_timestamp_raises_as_pyspark(protocol):
+    from pyspark.errors import PySparkValueError
+
+    frame = pd.DataFrame({"t": [pd.Timestamp("2020-01-01 
00:00:00.000000001")]})
+    with _arrow_input_service(protocol) as (spark, state):
+        with pytest.raises(PySparkValueError) as error:
+            spark.createDataFrame(frame, "t timestamp")
+    assert str(error.value) == (
+        "Exception thrown when converting pandas.Series (datetime64[ns, UTC]) 
with name 't' "
+        "to Arrow Array (timestamp[us, tz=UTC]). It can be caused by overflows 
or other unsafe "
+        "conversions warned by Arrow. Arrow safe type check can be disabled by 
using SQL config "
+        "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+    )
+
+
+def test_pandas_lone_surrogate_string_raises_as_pyspark(protocol):
+    from pyspark.errors import PySparkValueError
+
+    frame = pd.DataFrame({"s": ["a\ud800b"]}, dtype=object)
+    with _arrow_input_service(protocol) as (spark, state):
+        with pytest.raises(PySparkValueError) as error:
+            spark.createDataFrame(frame, "s string")
+    assert str(error.value) == (
+        "Exception thrown when converting pandas.Series (object) with name 's' 
to Arrow Array "
+        "(string). It can be caused by overflows or other unsafe conversions 
warned by Arrow. "
+        "Arrow safe type check can be disabled by using SQL config "
+        "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+    )
+
+
+def test_pandas_datetime_objects_in_date_column(protocol):
+    frame = pd.DataFrame({"d": [datetime(2020, 1, 2, 3, 4)]}, dtype=object)
+    table = _pandas_table(protocol, frame, "d date")
+    # Official: [Row(d=datetime.date(2020, 1, 2))].
+    assert table.column("d").type == pa.date32()
+    assert table.column("d").to_pylist() == [date(2020, 1, 2)]
+
+
+def test_pandas_nested_nan_becomes_null(protocol):
+    array_frame = pd.DataFrame({"s": [[float("nan"), 1.0]]}, dtype=object)
+    array_table = _pandas_table(protocol, array_frame, "s array<double>")
+    # Official: [Row(s=[None, 1.0])].
+    assert array_table.column("s").to_pylist() == [[None, 1.0]]
+
+    map_frame = pd.DataFrame({"s": [{"k": float("nan")}]}, dtype=object)
+    map_table = _pandas_table(protocol, map_frame, "s map<string,double>")
+    # Official: [Row(s={'k': None})].
+    assert map_table.column("s").to_pylist() == [[("k", None)]]
+
+
[email protected]("schema", [None, ["x"]], ids=["inferred", "names"])
+def test_pandas_inferred_int_beyond_int64_raises_as_pyspark(protocol, schema):
+    frame = pd.DataFrame({"x": [2**64]}, dtype=object)
+    with _arrow_input_service(protocol) as (spark, state):
+        # Official: OverflowError, not a double.
+        with pytest.raises(OverflowError) as error:
+            spark.createDataFrame(frame, schema)
+    assert str(error.value) == "Python int too large to convert to C long"
+
+
 # Exact built-in date columns reuse Arrow without materializing pandas rows.
 
 @pytest.mark.parametrize("cached", [False, True])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index afbc757..b489554 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5305,6 +5305,10 @@ def test_create_dataframe_rows_is_local_as_pyspark(data, 
schema, expected):
                                                   T.StructField("b", 
T.StringType())])])
 def test_create_dataframe_pandas_is_not_local_as_pyspark(schema):
     pd = pytest.importorskip("pandas")
+    if schema is not None:
+        # Explicit-schema pandas input reads the server's Arrow settings 
first, as PySpark
+        # does, so it needs a live server; the live-server suites cover it.
+        pytest.skip("explicit-schema pandas input reads the server 
configuration")
     # pyspark applies the columns, or the field names of the schema, with toDF.
     df = _local_create_dataframe(pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}), 
schema)
     assert not df.isLocal()
@@ -5665,18 +5669,14 @@ def 
test_local_input_native_number_values_and_types(mock_connect, value):
         assert actual == value
 
 
-def 
test_pandas_input_overflow_preserves_generic_conversion_behavior(mock_connect):
-    # This checks only that the exact-int fast path leaves integers outside 
the int64 range
-    # to the generic path. That path converts them to a double, which is a 
known difference
-    # from PySpark: it raises OverflowError ("Python int too large to convert 
to C long").
+def test_pandas_input_overflow_raises_as_pyspark(mock_connect):
+    # Without a schema, PySpark infers the Arrow type of an object column, so 
an integer
+    # outside the int64 range raises OverflowError rather than becoming a 
double.
     pd = pytest.importorskip("pandas")
     for value in (2**63, -(2**63) - 1):
-        actual = _native_number_input_table(
-            mock_connect, [Row(value=value)], source="pandas"
-        ).column(0)[0].as_py()
-        assert type(actual) is float
-        assert actual == float(value)
-    with pytest.raises(TypeError, match="unsupported value type.*int"):
+        with pytest.raises(OverflowError, match="^Python int too large to 
convert to C long$"):
+            _native_number_input_table(mock_connect, [Row(value=value)], 
source="pandas")
+    with pytest.raises(OverflowError, match="^Python int too large to convert 
to C long$"):
         mock_connect[0].createDataFrame(pd.DataFrame({"value": [2**20000]}, 
dtype=object))
 
 
@@ -5881,10 +5881,31 @@ def _container_input_capture_server(connection):
 
     with ThreadPoolExecutor(max_workers=2) as executor:
         server = grpc.server(executor)
+        def config(request, context):
+            # Explicit-schema pandas input reads the Arrow conversion settings 
first.
+            request = parse_message(request)
+            keys = [key.decode() for key in
+                    parse_message(parse_message(request[3][0])[2][0]).get(1, 
[])]
+            defaults = {
+                "spark.sql.timestampType": "TIMESTAMP_LTZ",
+                "spark.sql.session.timeZone": "UTC",
+                "spark.sql.execution.arrow.useLargeVarTypes": "false",
+                "spark.sql.session.localRelationCacheThreshold": str(1 << 60),
+                "spark.sql.session.localRelationSizeLimit": str(1 << 60),
+                "spark.sql.session.localRelationChunkSizeRows": str(1 << 30),
+                "spark.sql.session.localRelationChunkSizeBytes": str(1 << 40),
+                "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1 
<< 40),
+                "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
+            }
+            return message_field(1, request[1][0]) + b"".join(
+                message_field(2, message_field(1, key) + message_field(2, 
defaults[key]))
+                for key in keys)
+
         server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
             "spark.connect.SparkConnectService", {
                 "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute),
                 "AnalyzePlan": grpc.unary_unary_rpc_method_handler(_ddl_parse),
+                "Config": grpc.unary_unary_rpc_method_handler(config),
             }),))
         port = server.add_insecure_port("127.0.0.1:0")
         server.start()
@@ -6195,6 +6216,7 @@ def _exact_date_capture_server(connection):
         "spark.sql.session.localRelationChunkSizeRows": str(1 << 30),
         "spark.sql.session.localRelationChunkSizeBytes": str(1 << 40),
         "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1 << 40),
+        "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
     }
 
     def _exact_date_config(request, context):


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to