This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 78368f4  [SPARK-60697][CONNECT] Avoid DecimalTuple allocation for 
exact Decimal inputs
78368f4 is described below

commit 78368f4ed2bd8f10de27ce2d8c88a07c23ffb237
Author: DB Tsai <[email protected]>
AuthorDate: Sat Nov 7 11:21:48 2026 +0900

    [SPARK-60697][CONNECT] Avoid DecimalTuple allocation for exact Decimal 
inputs
    
    ### What changes were proposed in this pull request?
    
    Stacked on #132 (SPARK-60696); until it merges, this diff includes it.
    
    `createDataFrame` no longer builds a `DecimalTuple` (`Decimal.as_tuple()`, 
a tuple of every digit) for exact `decimal.Decimal` inputs:
    
    - `decimal_to_value` takes the scale of an exact Decimal from its text, 
`str()`, which #132 sends to the encoder anyway (`decimal_string_scale`).
    - The type of an inferred pandas column of Decimals, which is taken from 
all its values as Arrow does (`Decimal("123.4500")` gives `decimal(7, 4)`), 
comes from the same text of each exact Decimal (`decimal_text_digits`), instead 
of a second call of `as_tuple()` per cell.
    
    A subclass of `Decimal` can override `__str__` or `as_tuple`, so it is 
still asked for both, as Arrow does.
    
    ### Why are the changes needed?
    
    [SPARK-60697](https://issues.apache.org/jira/browse/SPARK-60697): the 
tuples were built only to read a digit count and an exponent that the text 
already has. Measured on macOS against a local Spark 4.3.0 server, as the 
client CPU time of `createDataFrame` of a pandas DataFrame of 200,000 rows of 
two Decimal columns (median of 5):
    
    | Schema | #132 | This change | PySpark 4.3.0 |
    | --- | --- | --- | --- |
    | Column names (types inferred) | 1,183 ms | 646 ms | 626 ms |
    | Explicit `decimal(12,4), decimal(10,2)` | 840 ms | 630 ms | 205 ms |
    
    All give the same types and the same sums of both columns.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, other than speed.
    
    ### How was this patch tested?
    
    - Offline tests in `python/tests/test_dropin_offline.py`, against #132's 
mock server that records the uploaded Arrow IPC: for local rows and pandas, 
with an explicit schema, column names or none, and fixed-point and scientific 
values, the IPC holds the cells and types of PySpark (an inferred pandas column 
typed by its values as `pa.array(values).type`, a local one `decimal(38, 18)`), 
whatever the Decimal context of the thread; subclasses are still asked for 
`__str__` and `as_tuple`, a [...]
    - Against the current master, with #132, and a local Spark 4.3.0 server: 
the Decimal tests of the offline suite (77 passed, 1 skipped); the full suites 
run before the merge.
    
    Changes on top of the original commits: the merge with master and #132's 
rework; inferring pandas Decimal columns from the text too, as master now types 
them by their values; and tests that expect PySpark's inferred types and share 
#132's fixture.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6); the rework on the current master by 
Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #133 from dbtsai/codex/spark-60697-decimal-tuple.
    
    Lead-authored-by: DB Tsai <[email protected]>
    Co-authored-by: Hyukjin Kwon <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/session.rs    |  96 ++++++++++++++++++++++++-------
 python/tests/test_dropin_offline.py | 109 ++++++++++++++++++++++++++++++++++++
 2 files changed, 185 insertions(+), 20 deletions(-)

diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index d17c8c3..a5d8cdd 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -652,7 +652,7 @@ impl PySparkSession {
                         .filter_map(|row| row.get(i))
                         .find(|value| !matches!(value, Value::Null));
                     let data_type = match first {
-                        Some(Value::Decimal { .. }) => 
pandas_decimal_type(&data, i)?,
+                        Some(Value::Decimal { .. }) => 
pandas_decimal_type(&data, &rows, i)?,
                         Some(value) => value_to_datatype(value),
                         None => 
pandas_dtype_type(dtypes.get(i).map_or("object", String::as_str)),
                     };
@@ -1687,21 +1687,37 @@ fn pandas_dtype_type(dtype: &str) -> DataType {
 
 /// The decimal type of column `i` of pandas rows, as Arrow infers it from all 
its Decimals:
 /// the most integer digits and the largest scale of the values.
-fn pandas_decimal_type(rows: &Bound<'_, PyList>, i: usize) -> 
PyResult<DataType> {
+fn pandas_decimal_type(data: &Bound<'_, PyList>, rows: &[Row], i: usize) -> 
PyResult<DataType> {
+    // The digits and exponent of an exact Decimal come from the text that 
`py_to_value` took
+    // of it, rather than from `as_tuple()`, which builds a DecimalTuple of 
every digit; a
+    // subclass, which can override either, is asked for `as_tuple()`, as 
Arrow does.
+    let decimal_cls = data.py().import("decimal")?.getattr("Decimal")?;
     let (mut integer_digits, mut scale) = (0i64, 0i64);
-    for row in rows.iter() {
+    for (row, converted) in data.iter().zip(rows) {
         let value = row.get_item(i)?;
         if value.is_none() {
             continue;
         }
-        let parts = value.call_method0("as_tuple")?;
-        let digits = parts.getattr("digits")?.len()? as i64;
-        // NaN and infinities have no exponent of digits, which Arrow cannot 
type either.
-        let Ok(exponent) = parts.getattr("exponent")?.extract::<i64>() else {
-            return Err(pyo3::exceptions::PyValueError::new_err(format!(
-                "Decimal {} has no precision to type a column by",
-                value.str()?
-            )));
+        let from_text = match converted.get(i) {
+            Some(Value::Decimal { value: text, .. }) if 
value.get_type().is(&decimal_cls) => {
+                decimal_text_digits(text)
+            }
+            _ => None,
+        };
+        let (digits, exponent) = match from_text {
+            Some(parts) => parts,
+            None => {
+                let parts = value.call_method0("as_tuple")?;
+                let digits = parts.getattr("digits")?.len()? as i64;
+                // NaN and infinities have no exponent of digits, which Arrow 
cannot type either.
+                let Ok(exponent) = parts.getattr("exponent")?.extract::<i64>() 
else {
+                    return Err(pyo3::exceptions::PyValueError::new_err(format!(
+                        "Decimal {} has no precision to type a column by",
+                        value.str()?
+                    )));
+                };
+                (digits, exponent)
+            }
         };
         let (precision, value_scale) = if exponent < 0 {
             (digits.max(-exponent), -exponent)
@@ -2050,7 +2066,7 @@ pub(crate) fn datetime_to_micros(obj: &Bound<'_, PyAny>) 
-> PyResult<i64> {
 }
 
 /// If `obj` is a `decimal.Decimal`, return it as a `Value::Decimal` (string 
value + scale
-/// from `as_tuple()`); otherwise `None`. Checked before int/float in 
`py_to_value` because
+/// from its exponent); otherwise `None`. Checked before int/float in 
`py_to_value` because
 /// Decimal defines `__float__`/`__int__`.
 fn decimal_to_value(obj: &Bound<'_, PyAny>) -> PyResult<Option<Value>> {
     let py = obj.py();
@@ -2066,20 +2082,60 @@ fn decimal_to_value(obj: &Bound<'_, PyAny>) -> 
PyResult<Option<Value>> {
     // either notation, in work bounded by the length of the text, which a 
fixed-point form
     // of `1E+1000000000` would not be.
     let dec_str: String = obj.str()?.extract()?;
-    let as_tuple = obj.call_method0("as_tuple")?;
-    let mut scale: Option<i32> = None;
-    if let Ok(exp_obj) = as_tuple.get_item(2) {
-        if !exp_obj.is_none() {
-            if let Ok(exp) = exp_obj.extract::<i64>() {
-                if exp <= 0 {
-                    scale = Some((-exp) as i32);
+    let scale = if obj.get_type().is(&decimal_cls) {
+        // Exact Decimal strings preserve the coefficient's trailing zeros and 
exponent.
+        // Subclasses may override either __str__ or as_tuple, so keep their 
old path.
+        decimal_string_scale(&dec_str)
+    } else {
+        let as_tuple = obj.call_method0("as_tuple")?;
+        let mut scale = None;
+        if let Ok(exp_obj) = as_tuple.get_item(2) {
+            if !exp_obj.is_none() {
+                if let Ok(exp) = exp_obj.extract::<i64>() {
+                    if exp <= 0 {
+                        scale = Some((-exp) as i32);
+                    }
                 }
             }
         }
-    }
+        scale
+    };
     Ok(Some(Value::Decimal {
         value: dec_str,
         precision: None,
         scale,
     }))
 }
+
+/// The scale of `Decimal.as_tuple()`, from the text of an exact Decimal: the 
negated exponent,
+/// or `None` for a positive exponent and for NaN and the infinities.
+fn decimal_string_scale(value: &str) -> Option<i32> {
+    let (_, exponent) = decimal_text_digits(value)?;
+    // As the scale of `as_tuple()`, including its i32 cast.
+    (exponent <= 0).then(|| (-exponent) as i32)
+}
+
+/// The number of digits and the exponent of `Decimal.as_tuple()`, from the 
text (`str()`) of
+/// an exact Decimal, which keeps every digit of its coefficient and its 
exponent: `123.4500`
+/// has 7 digits and the exponent -4, `1E+3` 1 and 3, `0.000` 1 and -3. `None` 
for NaN and the
+/// infinities, which have no such exponent.
+fn decimal_text_digits(value: &str) -> Option<(i64, i64)> {
+    let unsigned = value.trim_start_matches(['-', '+']);
+    if !unsigned.as_bytes().first()?.is_ascii_digit() {
+        return None;
+    }
+    let (coefficient, exponent) = match unsigned.split_once(['E', 'e']) {
+        Some((coefficient, exponent)) => (coefficient, 
exponent.parse::<i64>().ok()?),
+        None => (unsigned, 0),
+    };
+    let (integer, fraction) = 
coefficient.split_once('.').unwrap_or((coefficient, ""));
+    // The digits from the first nonzero one; zero has the single digit 0.
+    let leading_zeros = integer.len() - integer.trim_start_matches('0').len();
+    let leading_zeros = if leading_zeros == integer.len() {
+        leading_zeros + (fraction.len() - 
fraction.trim_start_matches('0').len())
+    } else {
+        leading_zeros
+    };
+    let digits = (integer.len() + fraction.len() - leading_zeros).max(1) as 
i64;
+    Some((digits, exponent - fraction.len() as i64))
+}
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index f9fc331..8db56e2 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6162,6 +6162,115 @@ def test_decimal_rescaling_follows_pyspark(
     assert not payload.exists()
 
 
[email protected]("kind", ["tuples", "pandas"])
[email protected]("schema_mode", ["explicit", "names", "inferred"])
[email protected]("texts", [
+    ["123.4500", "-2.3400", "0.0000", "-0.0000", None],
+    ["1.2300E+2", "-2.3400E+2", "0.00", "-0.00", None],
+    ["0.001234500", "-0.000002300", "0.000000", None],
+    ["123", "-4", "0", None],
+    ["1E+3", "1E-7", "-0E-7", "1.2300E-7", None],
+])
+def test_decimal_scale_input_ipc(decimal_ipc_session, kind, schema_mode, 
texts):
+    # The scale of a Decimal comes from its text, not from as_tuple(): the IPC 
holds the same
+    # cells and types as PySpark's, which types an inferred pandas column by 
its values as
+    # Arrow does, and a local one as decimal(38, 18). The Decimal context of 
the thread,
+    # here one that would round and trap, changes nothing.
+    from decimal import Decimal, Inexact, ROUND_UP, localcontext
+    import pyarrow as pa
+
+    spark, payload = decimal_ipc_session
+    values = [None if text is None else Decimal(text) for text in texts]
+    name = "_1" if schema_mode == "inferred" and kind == "tuples" else "v"
+    data = [(value,) for value in values]
+    if kind == "pandas":
+        pd = pytest.importorskip("pandas")
+        data = pd.DataFrame({name: values}, dtype=object)
+    if schema_mode == "explicit":
+        decimal_type = pa.decimal128(18, 9)
+        schema = T.StructType([T.StructField(name, T.DecimalType(18, 9))])
+    else:
+        decimal_type = pa.decimal128(38, 18) if kind == "tuples" else 
pa.array(values).type
+        schema = [name] if schema_mode == "names" else None
+    with localcontext() as context:
+        context.prec = 2
+        context.rounding = ROUND_UP
+        context.traps[Inexact] = True
+        spark.createDataFrame(data, schema).collect()
+    table = pa.ipc.open_stream(payload.read_bytes()).read_all()
+    expected = pa.table({name: pa.array(values, type=decimal_type)})
+    table.validate(full=True)
+    assert table.schema == expected.schema
+    assert table.equals(expected)
+
+
+def test_decimal_scale_of_subclasses(decimal_ipc_session):
+    # A subclass can override __str__ and as_tuple, so it is still asked for 
both, as Arrow
+    # does; their errors are raised.
+    from decimal import Decimal
+    import pyarrow as pa
+
+    pd = pytest.importorskip("pandas")
+    spark, payload = decimal_ipc_session
+    calls = []
+
+    class Traced(Decimal):
+        def __str__(self):
+            calls.append("str")
+            return super().__str__()
+
+        def as_tuple(self):
+            calls.append("tuple")
+            return super().as_tuple()
+
+    values = [Traced("1.20"), None, Traced("-30.5")]
+    spark.createDataFrame(pd.DataFrame({"v": values}, dtype=object), 
["v"]).collect()
+    assert calls.count("str") == 2 and "tuple" in calls
+    table = pa.ipc.open_stream(payload.read_bytes()).read_all()
+    expected = pa.array([Decimal("1.20"), None, Decimal("-30.5")])
+    assert table.column(0).combine_chunks().equals(expected)
+
+    for hook in ["__str__", "as_tuple"]:
+        def fail(self):
+            raise LookupError("decimal hook")
+
+        failing = type("Failing", (Decimal,), {hook: fail})
+        with pytest.raises(LookupError, match="decimal hook"):
+            spark.createDataFrame(pd.DataFrame({"v": [failing("1.5")]}, 
dtype=object), ["v"])
+
+
[email protected]("schema", [["v"], T.StructType([T.StructField("v", 
T.DecimalType(10, 4))])])
+def test_decimal_scale_avoids_coefficient_tuple(decimal_ipc_session, schema):
+    # Exact Decimals are typed and converted from their text: no DecimalTuple 
is built.
+    import cProfile
+    from decimal import Decimal
+
+    pd = pytest.importorskip("pandas")
+    spark, _ = decimal_ipc_session
+    tuple_constructor = type(Decimal("1.20").as_tuple()).__new__.__code__
+    values = pd.DataFrame({"v": [Decimal("123.4500"), Decimal("-2.3000"), 
None]}, dtype=object)
+    profile = cProfile.Profile()
+    profile.enable()
+    try:
+        spark.createDataFrame(values, schema)
+    finally:
+        profile.disable()
+    assert not any(entry.code is tuple_constructor for entry in 
profile.getstats())
+
+
[email protected]("text", ["Infinity", "-Infinity"])
+def test_decimal_scale_of_non_finite_values(decimal_ipc_session, text):
+    # An infinity has no precision to type a column by.
+    from decimal import Decimal
+
+    pd = pytest.importorskip("pandas")
+    spark, payload = decimal_ipc_session
+    data = pd.DataFrame({"v": [Decimal(text), Decimal("1.5")]}, dtype=object)
+    with pytest.raises(ValueError, match="has no precision to type a column 
by"):
+        spark.createDataFrame(data, ["v"])
+    assert not payload.exists()
+
+
 # --------------------------------------------------------------------------- 
schema cache
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to