This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new e22a657 [SPARK-61079][CONNECT][PYTHON] Convert pandas input with an
explicit schema as PySpark's Arrow conversion does
e22a657 is described below
commit e22a657e16b0cd46dad6a102ea2b9228ffe6f20a
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:50:23 2026 +0900
[SPARK-61079][CONNECT][PYTHON] Convert pandas input with an explicit schema
as PySpark's Arrow conversion does
### What changes were proposed in this pull request?
`createDataFrame` of a pandas DataFrame with an explicit schema (a
`StructType` or a DDL string) now converts it as PySpark 4.3.0 does: with the
vendored official `create_arrow_table_from_pandas` (`pyspark.sql.conversion`,
unchanged), after reading the Arrow options, the session time zone and
`spark.sql.execution.pandas.convertToArrowArraySafely` as PySpark does, and
sends the table through the Arrow importer, which uploads it at the relation
cache threshold (`create_from_pandas_arrow [...]
### Why are the changes needed?
[SPARK-61079](https://issues.apache.org/jira/browse/SPARK-61079): the row
conversion of pandas cells differed from PySpark's Arrow conversion. Against a
local Spark 4.3.0 server:
| Input | Current master | This change and PySpark 4.3.0 |
| --- | --- | --- |
| `{"f": [1, None]}` or `[1.0, nan]` into `f int` | `PySparkRuntimeError:
Type mismatch in row data` | `[Row(f=1), Row(f=None)]` |
| int64 `257` into `b tinyint` | `[Row(b=1)]`, silently |
`PySparkValueError` (Arrow's overflow) |
| a null in a non-nullable field | `PySparkRuntimeError: Failed to create
Arrow batch` | `ValueError: Casting field 'a' with null values to non-nullable`
|
| naive `2020-07-01 12:00`, client `TZ=America/Los_Angeles`, session UTC |
collects as 12:00 | collects as 05:00 (stored as 12:00 UTC); a DST gap raises
as PySpark |
| a timestamp with nanoseconds | truncated silently | `PySparkValueError` |
| a lone surrogate into `string` | `TypeError` | `PySparkValueError` |
| a `datetime` into `date` | `PySparkRuntimeError` | `date(2020, 1, 2)` |
| NaN nested in arrays and maps | kept as NaN | `None` |
| `2**64` in an inferred object column | a double | `OverflowError` |
### Does this PR introduce _any_ user-facing change?
Yes, these inputs convert, or fail, as in PySpark. Explicit-schema pandas
input reads its configs as PySpark does before converting.
### How was this patch tested?
- 30 new tests in `python/tests/test_arrow_handoff.py`, against the mock
Arrow input service, for each case above, the time-zone variants and the DST
gaps; existing tests that pinned the row path's errors or its config order now
expect PySpark's (listed in the commits).
- `test_arrow_handoff.py` (903 passed) and the pandas and createDataFrame
tests of the offline suite; each case above and both time-zone variants give
the same output as PySpark 4.3.0.
Still different: a frame whose width differs from the schema raises
`AXIS_LENGTH_MISMATCH`, where PySpark drops a column or raises `ArrowInvalid`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #270 from HyukjinKwon/SPARK-61079-pandas-explicit-schema.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 95 +++++++++++-
crates/spark-connect/src/session.rs | 7 +-
python/pyspark/sql/_pandas_input.py | 155 +++++++++++++++++++
python/tests/_mock_connect_server.py | 10 ++
python/tests/input_name_cases.py | 9 ++
python/tests/integer_tuple_cases.py | 15 +-
python/tests/test_arrow_handoff.py | 279 +++++++++++++++++++++++++++++++----
python/tests/test_dropin_offline.py | 42 ++++--
8 files changed, 566 insertions(+), 46 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index c6dc346..c62377a 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -544,6 +544,77 @@ impl PySparkSession {
}))
}
+ /// A DataFrame of a non-empty pandas DataFrame under an explicit struct
schema of the
+ /// same width, converted as PySpark does: each column by its Arrow
conversion (session
+ /// time zone, safe casts), then the table cast to the schema, which the
importer
+ /// sends as Arrow. None when the width differs, which the row path
reports.
+ fn create_from_pandas_arrow(
+ &self,
+ py: Python<'_>,
+ data: &Bound<'_, PyAny>,
+ schema: &Bound<'_, PyAny>,
+ fields: &[StructField],
+ ) -> PyResult<Option<PyDataFrame>> {
+ if fields.is_empty() || data.len()? == 0 ||
data.getattr("columns")?.len()? != fields.len()
+ {
+ return Ok(None);
+ }
+ // The caller's schema object is checked, before any configuration
request is sent.
+ let helper = py.import("pyspark.sql._pandas_input")?;
+ if !helper
+ .getattr("accepts_pandas_table")?
+ .call1((data, schema))?
+ .is_truthy()?
+ {
+ return Ok(None);
+ }
+ let options = py.detach(||
self.session.arrow_input_options().to_pyerr())?;
+ // The option is always set, as the importer requires it.
+ let timezone = options.timezone().unwrap_or("UTC").to_owned();
+ let large = options.use_large_var_types() == Some("true");
+ let safecheck = py
+ .detach(|| {
+ self.session
+ .conf()
+
.get("spark.sql.execution.pandas.convertToArrowArraySafely")
+ })
+ .to_pyerr()?
+ .as_deref()
+ == Some("true");
+ // The caller's StructType, which the guard checked, is converted as
given; a DDL
+ // string is parsed into the same fields.
+ let schema_py = if schema
+ .extract::<PyRef<crate::types::PyStructType>>()
+ .is_ok()
+ {
+ schema.clone().unbind()
+ } else {
+ crate::types::data_type_to_py(
+ py,
+ &DataType::Struct {
+ fields: fields.to_vec(),
+ },
+ )?
+ };
+ let table = helper.getattr("prepare_pandas_table")?.call1((
+ data,
+ schema_py.bind(py),
+ timezone,
+ safecheck,
+ large,
+ ))?;
+ let frame = crate::pyarrow_import::create_dataframe(
+ py,
+ &self.session,
+ &table,
+ Some(schema_py.bind(py)),
+ )?;
+ // As the other pandas paths, the explicit names are applied with toDF.
+ Ok(Some(PyDataFrame::new(frame.dataframe.to_df(
+ fields.iter().map(|field| field.name.as_str()).collect(),
+ ))))
+ }
+
/// A DataFrame of a pandas DataFrame: its rows as lists with NaN and NaT
as None, typed
/// by the first non-null value of each column, and its column labels, as
strings, or the
/// given names, applied with `toDF`.
@@ -553,6 +624,9 @@ impl PySparkSession {
data: &Bound<'_, PyAny>,
spec: Spec,
) -> PyResult<PyDataFrame> {
+ // Without a struct schema, PySpark infers the Arrow type of each
column, which has
+ // no int64 for a Python int beyond the 64-bit range.
+ let inferred = !matches!(spec, Spec::Struct(_));
let mut columns = Vec::new();
for label in data.getattr("columns")?.try_iter()? {
columns.push(label?.str()?.extract::<String>()?);
@@ -614,7 +688,19 @@ impl PySparkSession {
for row in data.iter() {
let values = row
.try_iter()?
- .map(|v| py_to_value(&v?))
+ .map(|v| {
+ let v = v?;
+ if inferred {
+ if let Ok(i) = v.cast_exact::<pyo3::types::PyInt>() {
+ if i.extract::<i64>().is_err() {
+ return
Err(pyo3::exceptions::PyOverflowError::new_err(
+ "Python int too large to convert to C
long",
+ ));
+ }
+ }
+ }
+ py_to_value(&v)
+ })
.collect::<PyResult<Vec<_>>>()?;
if values.len() != row_names.len() {
// As pyspark, which checks every row against the width of the
schema.
@@ -1159,6 +1245,13 @@ impl PySparkSession {
_ => frame,
});
}
+ // An explicit struct schema converts with PySpark's Arrow
conversion, which
+ // reports the same values and errors. Other schemas keep the row
path.
+ if let (Spec::Struct(fields), Some(schema)) = (&spec, schema) {
+ if let Some(frame) = self.create_from_pandas_arrow(py, data,
schema, fields)? {
+ return Ok(frame);
+ }
+ }
return self.create_from_pandas(py, data, spec);
}
// Preserve schema parsing, ignored-argument warnings, and
pandas/Arrow routing.
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 8e5201b..0b57655 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -986,9 +986,10 @@ fn row_width_mismatch(row_index: usize, actual: usize,
expected: usize) -> Spark
/// Convert Arrow RecordBatch data to IPC bytes.
/// Used by createDataFrame to package local data.
/// The size of a local relation, in Arrow data, from which `createDataFrame`
reads the
-/// server's local-relation settings to decide whether to upload it: half of
the default
-/// `spark.sql.session.localRelationCacheThreshold`, and well below the
default gRPC message
-/// size limit of the server (128 MiB). Smaller relations are sent inline
without that RPC.
+/// server's local-relation settings to decide whether to upload it, well
below the default
+/// gRPC message size limit of the server (128 MiB). Smaller relations are
sent inline without
+/// that RPC, even above `spark.sql.session.localRelationCacheThreshold` (1
MiB by default),
+/// at which PySpark uploads them.
const LOCAL_RELATION_SETTINGS_BYTES: usize = 32 << 20;
fn rows_to_arrow_batch(
diff --git a/python/pyspark/sql/_pandas_input.py
b/python/pyspark/sql/_pandas_input.py
index 2797219..f4891c2 100644
--- a/python/pyspark/sql/_pandas_input.py
+++ b/python/pyspark/sql/_pandas_input.py
@@ -813,6 +813,161 @@ def prepare_date_table(frame, schema):
)
+def _native_names_intact(namespace):
+ """Whether each public native class name in `namespace` is still the
native class."""
+ from pyspark import _pyspark as native
+
+ return all(
+ namespace[name] is value
+ for name, value in vars(native).items()
+ if not name.startswith("_") and isinstance(value, type) and name in
namespace
+ )
+
+
+def _canonical_struct(schema):
+ """Whether a native struct schema and its fields keep their own classes'
behavior.
+
+ The conversion reads the fields, names and data types through the classes,
so a
+ customized class keeps the row path.
+ """
+ from types import GetSetDescriptorType, WrapperDescriptorType
+
+ from pyspark._pyspark import StructField, StructType
+
+ if type(schema) is not StructType:
+ return False
+ for cls in (StructType, StructField):
+ if (cls.__getattribute__ is not object.__getattribute__
+ or any("__class__" in vars(base) for base in cls.__mro__ if
base is not object)):
+ return False
+ for cls, names, descriptor_type in (
+ (StructType, ("fields", "names"), GetSetDescriptorType),
+ (StructType, ("__iter__",), WrapperDescriptorType),
+ (StructField, ("name", "dataType", "nullable", "metadata"),
GetSetDescriptorType),
+ ):
+ for name in names:
+ descriptor = vars(cls).get(name)
+ if (type(descriptor) is not descriptor_type
+ or descriptor.__objclass__ is not cls or
descriptor.__name__ != name):
+ return False
+ return all(
+ type(field) is StructField and type(field.name) is str and "\x00" not
in field.name
+ and type(field.dataType).__getattribute__ is object.__getattribute__
+ for field in schema.fields
+ )
+
+
+def accepts_pandas_table(frame, schema):
+ """Whether `prepare_pandas_table` may convert this frame under this schema.
+
+ Customized frames, columns, schema classes and rebound type names keep the
row path,
+ which reads them in its own order. The conversion and the importer resolve
the type
+ names by lookup, so each namespace must still hold the native classes. The
schema is
+ the caller's object, or a DDL string, which is parsed.
+ """
+ import sys
+
+ import pandas as pd
+ from pandas.core.frame import DataFrame
+
+ from pyspark.sql import types as sql_types
+
+ if pd.DataFrame is not DataFrame or type(frame) is not DataFrame:
+ return False
+ if not _has_canonical_columns(frame.columns):
+ return False
+ # Custom extension arrays and dtypes keep the row path: Arrow would call
their own
+ # `__arrow_array__`, and their dtype may impersonate a NumPy type.
+ for i in range(len(frame.columns)):
+ column = frame.iloc[:, i]
+ if not all(type(value).__module__.split(".")[0] in ("pandas", "numpy")
+ for value in (column.array, column.dtype)):
+ return False
+ # The native row path normalizes cells through these methods, which a
customized class
+ # or instance keeps on the row path.
+ from pandas.core import frame as frame_module, generic as generic_module
+ from types import FunctionType
+
+ if any(name in vars(frame) for name in ("astype", "notna", "where")):
+ return False
+ for name, module, qualname in (
+ ("astype", generic_module, "NDFrame.astype"),
+ ("notna", frame_module, "DataFrame.notna"),
+ ("where", generic_module, "NDFrame.where"),
+ ):
+ method = getattr(DataFrame, name)
+ if (type(method) is not FunctionType or method.__qualname__ != qualname
+ or method.__module__ != module.__name__ or method.__globals__
is not vars(module)):
+ return False
+ if sys.modules.get("pyspark.sql.types") is not sql_types or not
_native_names_intact(
+ vars(sql_types)
+ ):
+ return False
+ for name in ("pyspark.sql.pandas.types", "pyspark.sql._arrow_input"):
+ module = sys.modules.get(name)
+ if module is not None and not _native_names_intact(vars(module)):
+ return False
+ # The importer resolves its schema classes by name, and its conversion
functions may
+ # keep the globals of an older type module.
+ importer = sys.modules.get("pyspark.sql._arrow_input")
+ if importer is not None:
+ from pyspark._pyspark import AtomicType, StructType
+
+ if any(vars(importer).get(name) is not cls
+ for name, cls in (("StructType", StructType), ("AtomicType",
AtomicType))):
+ return False
+ for function in list(vars(importer).values()):
+ namespace = getattr(function, "__globals__", None)
+ if type(namespace) is dict and not _native_names_intact(namespace):
+ return False
+ # The importer calls its converters by these names, which must still
be functions.
+ converter = sys.modules.get("pyspark.sql.pandas.types")
+ for name in ("to_arrow_schema", "from_arrow_schema"):
+ if name in vars(importer) and (
+ getattr(vars(importer)[name], "__globals__", None) is None
+ or (converter is not None
+ and vars(importer)[name] is not getattr(converter, name,
None))
+ ):
+ return False
+ return type(schema) is str or _canonical_struct(schema)
+
+
+def prepare_pandas_table(frame, schema, timezone, safecheck,
prefers_large_types):
+ """Convert a pandas frame under an explicit struct schema as PySpark
Connect does.
+
+ Each column goes through PySpark's own Arrow conversion, with the session
time zone
+ and the safe-cast setting, and the table is then renamed and cast to the
schema's
+ Arrow types. The values and errors therefore match PySpark, including the
nullability
+ and overflow messages. The caller checks the frame with
`accepts_pandas_table` and
+ the column count against the schema first.
+ """
+ from pyspark.sql.pandas.conversion import create_arrow_table_from_pandas
+ from pyspark.sql.pandas.types import _deduplicate_field_names,
to_arrow_schema
+
+ import pyarrow as pa
+
+ deduplicated = _deduplicate_field_names(schema)
+ arrow_schema = to_arrow_schema(
+ deduplicated, timezone="UTC", prefers_large_types=prefers_large_types
+ )
+ # Top-level names are the caller's, which may repeat: the importer casts
to them.
+ arrow_schema = pa.schema([
+ pa.field(name, field.type, field.nullable, field.metadata)
+ for name, field in zip(schema.names, arrow_schema)
+ ])
+ series_with_types = [
+ (column, field.dataType)
+ for (_, column), field in zip(frame.items(), deduplicated.fields)
+ ]
+ table = create_arrow_table_from_pandas(
+ series_with_types,
+ timezone=timezone,
+ safecheck=safecheck,
+ prefers_large_types=prefers_large_types,
+ )
+ return table.rename_columns(schema.names).cast(arrow_schema)
+
+
def arrow_decimal_column(values, dtype, name):
"""The precision, scale and values of a pandas object column of Decimals
and other values,
as Arrow types and converts it: PySpark's conversion of pandas leaves such
a column to
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
index 6822888..1699501 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -342,6 +342,16 @@ def serve(connection):
default_confs = {
"spark.sql.execution.pythonUDF.arrow.enabled": "true",
"spark.sql.execution.pythonUDTF.arrow.enabled": "true",
+ # Those that the Arrow conversion of pandas input reads, as a
Spark 4.3.0 server.
+ "spark.sql.session.timeZone": "UTC",
+ "spark.sql.timestampType": "TIMESTAMP_LTZ",
+ "spark.sql.execution.arrow.useLargeVarTypes": "false",
+ "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
+ "spark.sql.session.localRelationCacheThreshold": "1048576",
+ "spark.sql.session.localRelationSizeLimit": "3221225472",
+ "spark.sql.session.localRelationChunkSizeRows": "10000",
+ "spark.sql.session.localRelationChunkSizeBytes": "16777216",
+ "spark.sql.session.localRelationBatchOfChunksSizeBytes":
"1073741824",
}
read_confs = []
set_confs = []
diff --git a/python/tests/input_name_cases.py b/python/tests/input_name_cases.py
index 4bc5f43..6462b00 100644
--- a/python/tests/input_name_cases.py
+++ b/python/tests/input_name_cases.py
@@ -103,6 +103,15 @@ class InputNameTests(unittest.TestCase):
b"spark.sql.session.localRelationCacheThreshold": str(1 <<
60).encode(),
b"spark.sql.connect.serverStacktrace.enabled": b"false",
b"spark.sql.pyspark.jvmStacktrace.enabled": b"false",
+ # Explicit-schema pandas input reads the Arrow conversion
settings first.
+ b"spark.sql.timestampType": b"TIMESTAMP_LTZ",
+ b"spark.sql.session.timeZone": b"UTC",
+ b"spark.sql.execution.arrow.useLargeVarTypes": b"false",
+ b"spark.sql.session.localRelationSizeLimit": str(1 <<
60).encode(),
+ b"spark.sql.session.localRelationChunkSizeRows": str(1 <<
30).encode(),
+ b"spark.sql.session.localRelationChunkSizeBytes": str(1 <<
40).encode(),
+ b"spark.sql.session.localRelationBatchOfChunksSizeBytes":
str(1 << 40).encode(),
+ b"spark.sql.execution.pandas.convertToArrowArraySafely":
b"true",
}
pairs = b"".join(_bytes(2, _bytes(1, key) + _bytes(2, values[key]))
for key in keys)
diff --git a/python/tests/integer_tuple_cases.py
b/python/tests/integer_tuple_cases.py
index 184d4dc..2a6bd3f 100644
--- a/python/tests/integer_tuple_cases.py
+++ b/python/tests/integer_tuple_cases.py
@@ -102,6 +102,15 @@ class IntegerTupleTests(unittest.TestCase):
b"spark.sql.session.localRelationCacheThreshold": str(1 <<
60).encode(),
b"spark.sql.connect.serverStacktrace.enabled": b"false",
b"spark.sql.pyspark.jvmStacktrace.enabled": b"false",
+ # Explicit-schema pandas input reads the Arrow conversion
settings first.
+ b"spark.sql.timestampType": b"TIMESTAMP_LTZ",
+ b"spark.sql.session.timeZone": b"UTC",
+ b"spark.sql.execution.arrow.useLargeVarTypes": b"false",
+ b"spark.sql.session.localRelationSizeLimit": str(1 <<
60).encode(),
+ b"spark.sql.session.localRelationChunkSizeRows": str(1 <<
30).encode(),
+ b"spark.sql.session.localRelationChunkSizeBytes": str(1 <<
40).encode(),
+ b"spark.sql.session.localRelationBatchOfChunksSizeBytes":
str(1 << 40).encode(),
+ b"spark.sql.execution.pandas.convertToArrowArraySafely":
b"true",
}
pairs = b"".join(_bytes(2, _bytes(1, key) + _bytes(2, values[key]))
for key in keys)
@@ -134,7 +143,11 @@ class IntegerTupleTests(unittest.TestCase):
config_before = self.config_calls
frame = self.spark.createDataFrame(data, schema)
self.assertEqual(len(self.captured), before) # construction remains
offline
- self.assertEqual(self.config_calls, config_before)
+ # Explicit-schema pandas input reads the Arrow conversion settings
during
+ # construction, as PySpark does: the normalization and safe-cast
configuration.
+ expected = 3 if isinstance(data, pd.DataFrame) else 0
+ self.assertEqual(self.config_calls - config_before, expected)
+ config_before = self.config_calls
frame.toArrow()
self.assertEqual(len(self.captured), before + 1)
self.assertEqual(self.config_calls, config_before)
diff --git a/python/tests/test_arrow_handoff.py
b/python/tests/test_arrow_handoff.py
index 2f9f58b..b40a434 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -451,6 +451,7 @@ def _arrow_input_service(
"spark.sql.timestampType": "TIMESTAMP_LTZ",
"spark.sql.session.timeZone": "UTC",
"spark.sql.execution.arrow.useLargeVarTypes": "false",
+ "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
"spark.sql.session.localRelationCacheThreshold": "1048576",
"spark.sql.session.localRelationSizeLimit": str(3 * 1024**3),
"spark.sql.session.localRelationChunkSizeRows": "10000",
@@ -1555,12 +1556,19 @@ def test_non_nullable_schema_is_preserved(protocol,
verify, config_error):
frame = pd.DataFrame({"a": pd.Series([1, None], dtype="Int64")})
schema = T.StructType([T.StructField("a", T.LongType(), False)])
assert prepare_int64_table(frame, schema) is None
- with _arrow_input_service(protocol, config_error=config_error) as (spark,
state):
- with pytest.raises(RuntimeError, match="null"):
- spark.createDataFrame(frame, schema, verifySchema=verify)
+ with _arrow_input_service(protocol) as (spark, state):
+ if config_error:
+ state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+ with pytest.raises(Exception, match="configuration read failed"):
+ spark.createDataFrame(frame, schema, verifySchema=verify)
+ else:
+ # Official reads the configuration first, then the cast fails with
a ValueError.
+ with pytest.raises(ValueError) as error:
+ spark.createDataFrame(frame, schema, verifySchema=verify)
+ assert str(error.value) == "Casting field 'a' with null values to
non-nullable"
+ assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)
assert not state["relations"]
assert not state["uploads"]
- assert state["config_requests"] == []
def test_small_numpy_frames_avoid_importer_configuration_overhead():
@@ -1622,7 +1630,7 @@ def
test_integer_explicit_renamed_schema_preserves_all_ipc_cells(protocol, monke
from_arrow_schema(table.schema).json(), schema.json(),
table.schema)
assert state["config_requests"]
else:
- assert state["config_requests"] == []
+ assert state["config_requests"][:1] == [("get",
_NORMALIZATION_KEYS + _UPLOAD_KEYS)]
def test_integer_dataframe_and_name_subclasses_keep_fallback(monkeypatch):
@@ -2162,8 +2170,10 @@ def test_pandas_timestamp_non_nullable_schema(protocol,
has_null):
schema = T.StructType([T.StructField("a", T.TimestampType(), False)])
with _arrow_input_service(protocol, capture_tables=True) as (spark, state):
if has_null:
- with pytest.raises(RuntimeError, match="null"):
+ # Official: the cast of the Arrow table, a ValueError rather than
a row error.
+ with pytest.raises(ValueError) as error:
spark.createDataFrame(frame, schema)
+ assert str(error.value) == "Casting field 'a' with null values to
non-nullable"
assert not state["relations"]
else:
spark.createDataFrame(frame, schema).toArrow()
@@ -2184,17 +2194,25 @@ def
test_pandas_timestamp_mixed_and_arrow_dtypes_fallback():
@pytest.mark.parametrize("config_error", [False, True])
@pytest.mark.parametrize("verify", [False, True])
-def test_timestamp_nonnullable_nat_keeps_local_error_before_config(protocol,
config_error, verify):
+def test_timestamp_nonnullable_nat_reads_config_before_cast_error(protocol,
config_error, verify):
from pyspark.sql import types as T
from pyspark.sql._pandas_input import timestamp_columns
frame = pd.DataFrame({"a": pd.to_datetime([0, None], utc=True)})
schema = T.StructType([T.StructField("renamed", T.TimestampType(), False)])
assert timestamp_columns(frame, schema) is None
- with _arrow_input_service(protocol, capture_tables=True,
config_error=config_error) as (spark, state):
- with pytest.raises(RuntimeError, match="null"):
- spark.createDataFrame(frame, schema, verifySchema=verify)
- assert state["config_requests"] == []
+ with _arrow_input_service(protocol, capture_tables=True) as (spark, state):
+ if config_error:
+ state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+ with pytest.raises(Exception, match="configuration read failed"):
+ spark.createDataFrame(frame, schema, verifySchema=verify)
+ else:
+ # Official: ValueError from the cast of the Arrow table to the
schema.
+ with pytest.raises(ValueError) as error:
+ spark.createDataFrame(frame, schema, verifySchema=verify)
+ assert str(error.value) == (
+ "Casting field 'renamed' with null values to non-nullable")
+ assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)
assert state["relations"] == []
assert state["uploads"] == 0
@@ -2693,7 +2711,11 @@ def _assert_float64_ipc(table, columns, schema, *,
missing=None):
np.testing.assert_array_equal(actual.view(np.uint64),
expected[~mask].view(np.uint64))
actual_field = table.schema.field(index)
assert actual_field.nullable == field.nullable
- assert not actual_field.metadata
+ # The Arrow conversion keeps the field metadata; the fast path's table
has none.
+ from pyspark.sql.pandas.types import from_arrow_metadata
+
+ if actual_field.metadata is not None:
+ assert from_arrow_metadata(actual_field.metadata) == field.metadata
@@ -2755,8 +2777,10 @@ def
test_float64_columnar_ipc_matches_existing_row_conversion(protocol, monkeypa
assert len(state["input_tables"]) == 2
for table in state["input_tables"]:
_assert_float64_ipc(table, [values], schema)
- assert state["input_tables"][0].equals(state["input_tables"][1],
check_metadata=True)
- assert state["config_requests"] == []
+ # The Arrow conversion of the second call carries the field metadata
in its table
+ # and reads the configuration first, as official; the values and types
are the same.
+ assert state["input_tables"][0].equals(state["input_tables"][1],
check_metadata=False)
+ assert state["config_requests"][:1] == [("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)]
assert state["uploads"] == 0
assert [json.loads(relation.local_relation.schema) for relation in
state["relations"]] == [
json.loads(schema.json()), json.loads(schema.json())
@@ -2778,11 +2802,12 @@ def
test_float64_pandas_infinity_missing_option_keeps_row_semantics(protocol, nu
with pd.option_context("mode.use_inf_as_na", True):
with _arrow_input_service(protocol) as (spark, state):
if not nullable:
- with pytest.raises(PySparkRuntimeError, match=(
- "Column 'x' is declared as non-nullable but contains null
values"
- )):
+ # Official: the configuration is read, then the cast fails
with a ValueError.
+ with pytest.raises(ValueError) as error:
spark.createDataFrame(frame, schema)
- assert state["config_requests"] == []
+ assert str(error.value) == "Casting field 'x' with null values
to non-nullable"
+ assert state["config_requests"][:1] == [
+ ("get", _NORMALIZATION_KEYS + _UPLOAD_KEYS)]
assert state["uploads"] == 0
return
spark.createDataFrame(frame, schema).toArrow()
@@ -2901,26 +2926,31 @@ def
test_float64_public_fallback_preserves_every_ipc_cell(protocol, reason):
schema = expected_schema = T.StructType([T.StructField("x\x00renamed",
T.DoubleType())])
with _arrow_input_service(protocol) as (spark, state):
spark.createDataFrame(frame, schema).toArrow()
- assert state["config_requests"] == []
+ # Official converts explicit schemas with Arrow, reading the
configuration first.
+ native = reason in ("inferred", "names", "frame_subclass",
"schema_nul")
+ assert state["config_requests"][:1] == (
+ [] if native else [("get", _NORMALIZATION_KEYS + _UPLOAD_KEYS)])
_assert_float64_ipc(state["input_tables"][0], columns, expected_schema)
@pytest.mark.parametrize("verify", [False, True])
-def test_float64_nonnullable_nan_keeps_existing_error_before_configs(protocol,
verify):
- from pyspark.errors import PySparkRuntimeError
-
+def test_float64_nonnullable_nan_reads_config_before_cast_error(protocol,
verify):
frame = pd.DataFrame({"x": np.full(100000, np.nan, dtype=np.float64)})
schema = T.StructType([T.StructField("x", T.DoubleType(), False)])
with _arrow_input_service(protocol) as (spark, state):
state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
- with pytest.raises(PySparkRuntimeError, match=(
- "Column 'x' is declared as non-nullable but contains null values"
- )):
+ # Official reads the configuration first, so its error precedes the
cast.
+ with pytest.raises(Exception, match="configuration read failed"):
spark.createDataFrame(frame, schema, verifySchema=verify)
- assert state["config_requests"] == []
+ assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)
assert state["relations"] == []
assert state["uploads"] == 0
+ # Then the cast fails as official does, with a ValueError.
+ with pytest.raises(ValueError) as error:
+ spark.createDataFrame(frame, schema, verifySchema=verify)
+ assert str(error.value) == "Casting field 'x' with null values to
non-nullable"
+ assert state["relations"] == []
@@ -3303,7 +3333,7 @@ def
test_float64_missing_concrete_numpy_backing_class_keeps_fallback(monkeypatch
@pytest.mark.parametrize("rows", [10000, 99999, 100000])
-def test_float64_public_threshold_preserves_row_path_below_100k(protocol,
rows):
+def test_float64_public_threshold_reads_config_below_100k(protocol, rows):
import json
values = np.resize(np.array([np.nan, -0.0, 0.0, np.inf, -np.inf, 1.5]),
rows)
@@ -3314,12 +3344,21 @@ def
test_float64_public_threshold_preserves_row_path_below_100k(protocol, rows):
with _arrow_input_service(
protocol, **{"spark.sql.session.localRelationCacheThreshold":
"999999999"}
) as (spark, state):
+ if rows >= 100000:
+ # The float64 fast path converts at this size without a
configuration read.
+ spark.createDataFrame(frame, schema).toArrow()
+ _assert_float64_ipc(state["input_tables"][0], [values], schema)
+ assert state["config_requests"] == []
+ return
state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT
+ # Official reads the configuration before converting, so its error
comes first.
+ with pytest.raises(Exception, match="configuration read failed"):
+ spark.createDataFrame(frame, schema)
spark.createDataFrame(frame, schema).toArrow()
_assert_float64_ipc(state["input_tables"][0], [values], schema)
assert json.loads(state["relations"][0].local_relation.schema) ==
json.loads(schema.json())
assert state["uploads"] == 0
- assert state["config_requests"] == []
+ assert state["config_requests"][0] == ("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)
@@ -4198,7 +4237,6 @@ def
test_string_public_fallback_keeps_row_semantics(protocol, monkeypatch, reaso
assert state["config_requests"] == []
-
@pytest.mark.parametrize("surrogate", ["\ud800", "\udfff"])
def test_string_surrogate_fallback_precedes_importer_configuration(protocol,
monkeypatch, surrogate):
values = ["plain"] * 10000
@@ -4223,7 +4261,6 @@ def
test_string_surrogate_fallback_precedes_importer_configuration(protocol, mon
assert state["uploads"] == 0
-
@pytest.mark.parametrize("class_name,attribute", [
("StructType", "names"), ("StructType", "fields"), ("StructType",
"__iter__"),
("StructField", "name"), ("StructField", "dataType"), ("StructField",
"nullable"),
@@ -4284,7 +4321,12 @@ class Factory:
return Path(tempfile.mkdtemp(prefix=name))
with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state):
spark.createDataFrame(frame, schema).toArrow()
- assert state["config_requests"] == [], state["config_requests"]
+ if attribute == "isUTF8BinaryCollation":
+ # Official converts this explicit schema with Arrow, after reading the
configuration;
+ # it never reads this attribute, so the result is the same as the row
path's.
+ assert state["config_requests"][0][0] == "get",
state["config_requests"]
+ else:
+ assert state["config_requests"] == [], state["config_requests"]
table = state["input_tables"][0]
assert table.schema.names == ["text"], table.schema
assert table.column(0).to_pylist() == ["plain"] * 10000
@@ -4543,6 +4585,181 @@ def
test_string_large_inline_input_keeps_row_transport_and_schema(protocol):
__import__("json").loads(schema.json()))
+# Explicit-schema pandas input converts as PySpark's Arrow conversion does:
the session time
+# zone, the safe-cast setting, the nullability and the errors. Each
expectation below was
+# checked against official PySpark 4.3.0 with the same frame, schema and
session settings.
+
+
+def _pandas_table(protocol, frame, schema, **overrides):
+ """The table that an explicit-schema pandas frame sends, and whether it is
local."""
+ with _arrow_input_service(protocol, **overrides) as (spark, state):
+ df = spark.createDataFrame(frame, schema)
+ assert not df.isLocal()
+ df.toArrow()
+ return state["input_tables"][0]
+
+
+def _in_host_zone(name, call):
+ """Run `call` with the process time zone `name`, as the client's local
time zone."""
+ import time
+
+ previous = os.environ.get("TZ")
+ os.environ["TZ"] = name
+ time.tzset()
+ try:
+ return call()
+ finally:
+ if previous is None:
+ os.environ.pop("TZ", None)
+ else:
+ os.environ["TZ"] = previous
+ time.tzset()
+
+
[email protected]("values", [[1, None], [1.0, np.nan]], ids=["int",
"float"])
+def test_pandas_explicit_int_nulls_are_null(protocol, values):
+ table = _pandas_table(protocol, pd.DataFrame({"f": values}), "f int")
+ # Official: [Row(f=1), Row(f=None)].
+ assert table.schema.field("f").type == pa.int32()
+ assert table.column("f").to_pylist() == [1, None]
+
+
+def _tinyint_schema():
+ return T.StructType([T.StructField("b", T.ByteType())])
+
+
+def test_pandas_explicit_tinyint_overflow_raises_as_pyspark(protocol):
+ from pyspark.errors import PySparkValueError
+
+ with _arrow_input_service(protocol) as (spark, state):
+ with pytest.raises(PySparkValueError) as error:
+ spark.createDataFrame(pd.DataFrame({"b": [257]}),
_tinyint_schema())
+ assert str(error.value) == (
+ "Exception thrown when converting pandas.Series (int64) with name 'b'
to Arrow Array "
+ "(int8). It can be caused by overflows or other unsafe conversions
warned by Arrow. "
+ "Arrow safe type check can be disabled by using SQL config "
+ "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+ )
+ assert state["input_tables"] == []
+
+
+def test_pandas_explicit_tinyint_overflow_wraps_when_unsafe(protocol):
+ # Official, with convertToArrowArraySafely=false: [Row(b=1)].
+ table = _pandas_table(
+ protocol, pd.DataFrame({"b": [257]}), _tinyint_schema(),
+ **{"spark.sql.execution.pandas.convertToArrowArraySafely": "false"},
+ )
+ assert table.column("b").to_pylist() == [1]
+
+
+def test_pandas_explicit_nonnullable_null_raises_as_pyspark(protocol):
+ schema = T.StructType([T.StructField("a", T.LongType(), False)])
+ frame = pd.DataFrame({"a": pd.array([1, None], dtype="Int64")})
+ with _arrow_input_service(protocol) as (spark, state):
+ with pytest.raises(ValueError) as error:
+ spark.createDataFrame(frame, schema)
+ # Official: ValueError, not a PySparkRuntimeError from the row path.
+ assert type(error.value) is ValueError
+ assert str(error.value) == "Casting field 'a' with null values to
non-nullable"
+ assert state["input_tables"] == []
+
+
[email protected]("host", ["America/Los_Angeles", "UTC"])
+def test_pandas_naive_timestamp_uses_session_zone(protocol, host):
+ frame = pd.DataFrame({"t": [pd.Timestamp("2020-07-01 12:00")]})
+ # Official with session UTC, whatever the client zone: 12:00 UTC.
+ table = _in_host_zone(
+ host,
+ lambda: _pandas_table(
+ protocol, frame, "t timestamp",
+ **{"spark.sql.session.timeZone": "UTC"},
+ ),
+ )
+ assert table.column("t").to_pylist() == [datetime(2020, 7, 1, 12,
tzinfo=timezone.utc)]
+
+
+def test_pandas_naive_timestamp_in_session_zone_is_converted(protocol):
+ pytz = pytest.importorskip("pytz")
+ frame = pd.DataFrame({"t": [pd.Timestamp("2020-07-01 12:00")]})
+ # Official with session America/Los_Angeles and a client zone of UTC:
19:00 UTC.
+ table = _in_host_zone(
+ "UTC",
+ lambda: _pandas_table(
+ protocol, frame, "t timestamp",
+ **{"spark.sql.session.timeZone": "America/Los_Angeles"},
+ ),
+ )
+ assert table.column("t").to_pylist() == [datetime(2020, 7, 1, 19,
tzinfo=timezone.utc)]
+
+ gap = pd.DataFrame({"t": [pd.Timestamp("2020-03-08 02:30")]})
+ with _arrow_input_service(protocol, **{
+ "spark.sql.session.timeZone": "America/Los_Angeles",
+ }) as (spark, _):
+ # Official: pytz NonExistentTimeError for a naive time inside the DST
gap.
+ with pytest.raises(pytz.exceptions.NonExistentTimeError):
+ spark.createDataFrame(gap, "t timestamp")
+
+
+def test_pandas_nanosecond_timestamp_raises_as_pyspark(protocol):
+ from pyspark.errors import PySparkValueError
+
+ frame = pd.DataFrame({"t": [pd.Timestamp("2020-01-01
00:00:00.000000001")]})
+ with _arrow_input_service(protocol) as (spark, state):
+ with pytest.raises(PySparkValueError) as error:
+ spark.createDataFrame(frame, "t timestamp")
+ assert str(error.value) == (
+ "Exception thrown when converting pandas.Series (datetime64[ns, UTC])
with name 't' "
+ "to Arrow Array (timestamp[us, tz=UTC]). It can be caused by overflows
or other unsafe "
+ "conversions warned by Arrow. Arrow safe type check can be disabled by
using SQL config "
+ "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+ )
+
+
+def test_pandas_lone_surrogate_string_raises_as_pyspark(protocol):
+ from pyspark.errors import PySparkValueError
+
+ frame = pd.DataFrame({"s": ["a\ud800b"]}, dtype=object)
+ with _arrow_input_service(protocol) as (spark, state):
+ with pytest.raises(PySparkValueError) as error:
+ spark.createDataFrame(frame, "s string")
+ assert str(error.value) == (
+ "Exception thrown when converting pandas.Series (object) with name 's'
to Arrow Array "
+ "(string). It can be caused by overflows or other unsafe conversions
warned by Arrow. "
+ "Arrow safe type check can be disabled by using SQL config "
+ "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+ )
+
+
+def test_pandas_datetime_objects_in_date_column(protocol):
+ frame = pd.DataFrame({"d": [datetime(2020, 1, 2, 3, 4)]}, dtype=object)
+ table = _pandas_table(protocol, frame, "d date")
+ # Official: [Row(d=datetime.date(2020, 1, 2))].
+ assert table.column("d").type == pa.date32()
+ assert table.column("d").to_pylist() == [date(2020, 1, 2)]
+
+
+def test_pandas_nested_nan_becomes_null(protocol):
+ array_frame = pd.DataFrame({"s": [[float("nan"), 1.0]]}, dtype=object)
+ array_table = _pandas_table(protocol, array_frame, "s array<double>")
+ # Official: [Row(s=[None, 1.0])].
+ assert array_table.column("s").to_pylist() == [[None, 1.0]]
+
+ map_frame = pd.DataFrame({"s": [{"k": float("nan")}]}, dtype=object)
+ map_table = _pandas_table(protocol, map_frame, "s map<string,double>")
+ # Official: [Row(s={'k': None})].
+ assert map_table.column("s").to_pylist() == [[("k", None)]]
+
+
[email protected]("schema", [None, ["x"]], ids=["inferred", "names"])
+def test_pandas_inferred_int_beyond_int64_raises_as_pyspark(protocol, schema):
+ frame = pd.DataFrame({"x": [2**64]}, dtype=object)
+ with _arrow_input_service(protocol) as (spark, state):
+ # Official: OverflowError, not a double.
+ with pytest.raises(OverflowError) as error:
+ spark.createDataFrame(frame, schema)
+ assert str(error.value) == "Python int too large to convert to C long"
+
+
# Exact built-in date columns reuse Arrow without materializing pandas rows.
@pytest.mark.parametrize("cached", [False, True])
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index afbc757..b489554 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5305,6 +5305,10 @@ def test_create_dataframe_rows_is_local_as_pyspark(data,
schema, expected):
T.StructField("b",
T.StringType())])])
def test_create_dataframe_pandas_is_not_local_as_pyspark(schema):
pd = pytest.importorskip("pandas")
+ if schema is not None:
+ # Explicit-schema pandas input reads the server's Arrow settings
first, as PySpark
+ # does, so it needs a live server; the live-server suites cover it.
+ pytest.skip("explicit-schema pandas input reads the server
configuration")
# pyspark applies the columns, or the field names of the schema, with toDF.
df = _local_create_dataframe(pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}),
schema)
assert not df.isLocal()
@@ -5665,18 +5669,14 @@ def
test_local_input_native_number_values_and_types(mock_connect, value):
assert actual == value
-def
test_pandas_input_overflow_preserves_generic_conversion_behavior(mock_connect):
- # This checks only that the exact-int fast path leaves integers outside
the int64 range
- # to the generic path. That path converts them to a double, which is a
known difference
- # from PySpark: it raises OverflowError ("Python int too large to convert
to C long").
+def test_pandas_input_overflow_raises_as_pyspark(mock_connect):
+ # Without a schema, PySpark infers the Arrow type of an object column, so
an integer
+ # outside the int64 range raises OverflowError rather than becoming a
double.
pd = pytest.importorskip("pandas")
for value in (2**63, -(2**63) - 1):
- actual = _native_number_input_table(
- mock_connect, [Row(value=value)], source="pandas"
- ).column(0)[0].as_py()
- assert type(actual) is float
- assert actual == float(value)
- with pytest.raises(TypeError, match="unsupported value type.*int"):
+ with pytest.raises(OverflowError, match="^Python int too large to
convert to C long$"):
+ _native_number_input_table(mock_connect, [Row(value=value)],
source="pandas")
+ with pytest.raises(OverflowError, match="^Python int too large to convert
to C long$"):
mock_connect[0].createDataFrame(pd.DataFrame({"value": [2**20000]},
dtype=object))
@@ -5881,10 +5881,31 @@ def _container_input_capture_server(connection):
with ThreadPoolExecutor(max_workers=2) as executor:
server = grpc.server(executor)
+ def config(request, context):
+ # Explicit-schema pandas input reads the Arrow conversion settings
first.
+ request = parse_message(request)
+ keys = [key.decode() for key in
+ parse_message(parse_message(request[3][0])[2][0]).get(1,
[])]
+ defaults = {
+ "spark.sql.timestampType": "TIMESTAMP_LTZ",
+ "spark.sql.session.timeZone": "UTC",
+ "spark.sql.execution.arrow.useLargeVarTypes": "false",
+ "spark.sql.session.localRelationCacheThreshold": str(1 << 60),
+ "spark.sql.session.localRelationSizeLimit": str(1 << 60),
+ "spark.sql.session.localRelationChunkSizeRows": str(1 << 30),
+ "spark.sql.session.localRelationChunkSizeBytes": str(1 << 40),
+ "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1
<< 40),
+ "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
+ }
+ return message_field(1, request[1][0]) + b"".join(
+ message_field(2, message_field(1, key) + message_field(2,
defaults[key]))
+ for key in keys)
+
server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
"spark.connect.SparkConnectService", {
"ExecutePlan": grpc.unary_stream_rpc_method_handler(execute),
"AnalyzePlan": grpc.unary_unary_rpc_method_handler(_ddl_parse),
+ "Config": grpc.unary_unary_rpc_method_handler(config),
}),))
port = server.add_insecure_port("127.0.0.1:0")
server.start()
@@ -6195,6 +6216,7 @@ def _exact_date_capture_server(connection):
"spark.sql.session.localRelationChunkSizeRows": str(1 << 30),
"spark.sql.session.localRelationChunkSizeBytes": str(1 << 40),
"spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1 << 40),
+ "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
}
def _exact_date_config(request, context):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]