This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new db11b88  [SPARK-61071][CONNECT][PYTHON] Type a pandas column of 
Decimals and other values as Arrow does, as PySpark does
db11b88 is described below

commit db11b88d512202b54c38afcd1a442116a4958a8b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:27:05 2026 +0900

    [SPARK-61071][CONNECT][PYTHON] Type a pandas column of Decimals and other 
values as Arrow does, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    The type of an inferred pandas object column that mixes Decimal values with 
other values now comes from Arrow, as in PySpark, whose conversion of pandas 
leaves such a column to Arrow (`arrow_decimal_column` in 
`python/pyspark/sql/_pandas_input.py`, called from `create_from_pandas`): Arrow 
types the column, takes ints as Decimals, and raises for the others, which 
become PySpark's `PySparkTypeError` or `PySparkValueError` ("Exception thrown 
when converting pandas.Series (object) with na [...]
    
    ### Why are the changes needed?
    
    [SPARK-61071](https://issues.apache.org/jira/browse/SPARK-61071): such a 
column raised `AttributeError: 'int' object has no attribute 'as_tuple'`, or 
typed a column whose first value is an int as bigint and then failed. Against a 
local Spark 4.3.0 server, these now give the same results as PySpark 4.3.0:
    
    | Column | Current master | This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `[Decimal("1.5"), 2]` | `AttributeError` | `decimal(2,1)`, `[1.5, 2.0]` |
    | `[2, Decimal("1.5")]` | `PySparkRuntimeError: Type mismatch in row data` 
| `decimal(2,1)`, `[2.0, 1.5]` |
    | `[Decimal("1.5"), 2.5]` or `"x"` | `AttributeError` | `PySparkTypeError` |
    | `[Decimal("1.5"), True]`, `[Decimal("-0.5"), -3, None]`, 
`[Decimal("1.25"), 10**20]` | `AttributeError` | `PySparkValueError` |
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these columns convert, or fail, as in PySpark.
    
    ### How was this patch tested?
    
    - A new offline test in `python/tests/test_dropin_offline.py` for each case 
above, against the mock server that records the uploaded Arrow IPC (types and 
cells), and the Decimal tests of the offline suite.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #265 from HyukjinKwon/SPARK-61071-mixed-decimals.
    
    Lead-authored-by: Hyukjin Kwon <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    | 46 +++++++++++++++++++++++++++++++++----
 python/pyspark/sql/_pandas_input.py | 24 +++++++++++++++++++
 python/tests/test_dropin_offline.py | 36 +++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 4 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 2b05a00..a6694fd 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -656,10 +656,48 @@ impl PySparkSession {
                         .iter()
                         .filter_map(|row| row.get(i))
                         .find(|value| !matches!(value, Value::Null));
-                    let data_type = match first {
-                        Some(Value::Decimal { .. }) => 
pandas_decimal_type(&data, &rows, i)?,
-                        Some(value) => value_to_datatype(value),
-                        None => 
pandas_dtype_type(dtypes.get(i).map_or("object", String::as_str)),
+                    let cells = || rows.iter().filter_map(|row| row.get(i));
+                    let data_type = if !cells().any(|v| matches!(v, 
Value::Decimal { .. })) {
+                        match first {
+                            Some(value) => value_to_datatype(value),
+                            None => {
+                                
pandas_dtype_type(dtypes.get(i).map_or("object", String::as_str))
+                            }
+                        }
+                    } else if cells().all(|v| matches!(v, Value::Decimal { .. 
} | Value::Null)) {
+                        pandas_decimal_type(&data, &rows, i)?
+                    } else {
+                        // Decimals with other values, which PySpark's 
conversion leaves to
+                        // Arrow: it types the column, takes ints as Decimals 
and raises for
+                        // the others, so the column is typed and converted by 
Arrow here too.
+                        let column = data
+                            .iter()
+                            .map(|row| row.get_item(i))
+                            .collect::<PyResult<Vec<_>>>()?;
+                        let typed = py
+                            .import("pyspark.sql._pandas_input")?
+                            .getattr("arrow_decimal_column")?
+                            .call1((
+                                column,
+                                dtypes.get(i).map_or("object", String::as_str),
+                                labels.get(i).unwrap_or(name),
+                            ))?;
+                        let (precision, scale, values): (i32, i32, 
Vec<Bound<'_, PyAny>>) =
+                            typed.extract()?;
+                        let values = values
+                            .iter()
+                            .map(|v| Ok(py_to_value(v)?))
+                            .collect::<PyResult<Vec<Value>>>()?;
+                        rows = std::mem::take(&mut rows)
+                            .into_iter()
+                            .zip(values)
+                            .map(|(row, value)| {
+                                let mut cells = row.into_values();
+                                cells[i] = value;
+                                StructValue::new(row_names.clone(), 
cells).into()
+                            })
+                            .collect();
+                        DataType::Decimal { precision, scale }
                     };
                     fields.push(StructField {
                         name: name.clone(),
diff --git a/python/pyspark/sql/_pandas_input.py 
b/python/pyspark/sql/_pandas_input.py
index 1077dd4..2797219 100644
--- a/python/pyspark/sql/_pandas_input.py
+++ b/python/pyspark/sql/_pandas_input.py
@@ -811,3 +811,27 @@ def prepare_date_table(frame, schema):
         [pa.array(column, type=pa.date32(), from_pandas=False) for column in 
columns],
         names=schema.names,
     )
+
+
+def arrow_decimal_column(values, dtype, name):
+    """The precision, scale and values of a pandas object column of Decimals 
and other values,
+    as Arrow types and converts it: PySpark's conversion of pandas leaves such 
a column to
+    Arrow, which takes ints as Decimals and raises for the others."""
+    import pyarrow as pa
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+
+    message = ("Exception thrown when converting pandas.Series (%s) with name 
'%s' to Arrow "
+               "Array (None)." % (dtype, name))
+    try:
+        array = pa.array(values, from_pandas=True)
+    except pa.lib.ArrowException as e:
+        if isinstance(e, TypeError):
+            raise PySparkTypeError(message) from e
+        raise PySparkValueError(
+            message + " It can be caused by overflows or other unsafe 
conversions warned by "
+            "Arrow. Arrow safe type check can be disabled by using SQL config "
+            "`spark.sql.execution.pandas.convertToArrowArraySafely`.") from e
+    if not pa.types.is_decimal(array.type):
+        raise PySparkTypeError(message)
+    return array.type.precision, array.type.scale, array.to_pylist()
+
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index c3b18d0..9d7152f 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5814,6 +5814,42 @@ def 
test_container_input_nested_array_struct_binary_ipc(container_input_capture_
                                   "record": {"x": 7, "y": "row"}, "blob": 
b"abc"}]
 
 
+# --------------------------------------------------------------------------- 
pandas columns of Decimals and other values
+
+
[email protected]("values, expected", [
+    (["1.5", 2], (2, 1, ["1.5", "2.0"])),
+    ([2, "1.5"], (2, 1, ["2.0", "1.5"])),
+    (["1.5", 2.5], "PySparkTypeError"),
+    (["1.5", ("x",)], "PySparkTypeError"),
+    (["1.5", True], "PySparkValueError"),
+    (["-0.5", -3, None], "PySparkValueError"),
+])
+def 
test_pandas_columns_of_decimals_and_other_values_follow_arrow(decimal_ipc_session,
 values, expected):
+    # PySpark's conversion leaves such a column to Arrow, which types it, 
takes ints as
+    # Decimals and raises for the others (strings in `values` are Decimals 
here, and a 1-tuple
+    # holds a plain string).
+    from decimal import Decimal
+    import pyarrow as pa
+    from pyspark import errors
+
+    pd = pytest.importorskip("pandas")
+    spark, payload = decimal_ipc_session
+    cells = [Decimal(v) if isinstance(v, str) else v[0] if isinstance(v, 
tuple) else v
+             for v in values]
+    data = pd.DataFrame({"d": cells}, dtype=object)
+    if isinstance(expected, str):
+        with pytest.raises(getattr(errors, expected), match=r"pandas.Series 
\(object\) with name 'd'"):
+            spark.createDataFrame(data)
+        assert not payload.exists()
+        return
+    precision, scale, texts = expected
+    spark.createDataFrame(data).collect()
+    table = pa.ipc.open_stream(payload.read_bytes()).read_all()
+    assert table.column(0).type == pa.decimal128(precision, scale)
+    assert table.column(0).to_pylist() == [Decimal(text) for text in texts]
+
+
 # --------------------------------------------------------------------------- 
exact date input
 
 from datetime import (


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to