This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new e541f64  [SPARK-60994][CONNECT][PYTHON] Type a decimal literal as 
PySpark does
e541f64 is described below

commit e541f64c94e448d4b109686a3397469fbaf9c5cd
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 16:18:01 2026 +0900

    [SPARK-60994][CONNECT][PYTHON] Type a decimal literal as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60994](https://issues.apache.org/jira/browse/SPARK-60994): a literal 
of a Python `Decimal` now has the precision 10 and scale 0 of `DecimalType()`, 
as PySpark's `LiteralExpression` of a `Decimal` does. That covers `lit()` and 
any other Column built from one, such as `F.col("a") == Decimal("1.5")`. The 
server widens them to the precision and scale of the value. Before, the literal 
had the precision and scale of the value itself.
    
    ### Why are the changes needed?
    
    The literals, and the expressions over them, had other types than in 
PySpark. Against a Spark 4.3.0 server:
    
    | Column | PySpark 4.3.0 and now | Before |
    | --- | --- | --- |
    | `lit(Decimal("1.5"))` | `decimal(10,1)` | `decimal(2,1)` |
    | `lit(Decimal(0))` | `decimal(10,0)` | `decimal(1,0)` |
    | `lit(Decimal("-0.001"))` | `decimal(10,3)` | `decimal(3,3)` |
    | `lit(Decimal("1.5")) * lit(Decimal("2.25"))` | `decimal(21,3)` | 
`decimal(6,3)` |
    
    So an expression could overflow where PySpark's does not. A literal of a 
value with more than 10 digits, such as `Decimal("123456789012.5")`, keeps its 
own precision in both.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, decimal literals have the types of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, the types and values of the literals 
above give the same results as official PySpark 4.3.0.
    - A new offline test checks, through a mock Connect server, the decimal 
literal that `lit()` and a comparison send. A new live test checks the types 
and values above. It also passes with official PySpark 4.3.0. Both fail without 
this change.
    - The lib tests and every integration test, e2e included, against the local 
server (648 passed, and 123 of the transport crate), the build of all tests 
with `wasm-udf`, the offline Python suites (721 passed, 9 skipped), and the 
live drop-in and Arrow suites (77 passed) and live cases (10 passed). Clippy 
reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #196 from HyukjinKwon/SPARK-60994-decimal-literal.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs  | 15 ++++++---------
 python/tests/test_dropin_e2e.py     | 21 +++++++++++++++++++++
 python/tests/test_dropin_offline.py | 27 +++++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index 6f24de8..392357b 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -261,19 +261,16 @@ fn scalar_literal(obj: &Bound<'_, PyAny>) -> 
PyResult<Option<LiteralExpression>>
     let py = obj.py();
 
     // decimal.Decimal -> Decimal literal (before int/float: Decimal has 
__float__/__int__).
+    // As PySpark's LiteralExpression of a Decimal, of DecimalType(): 
precision 10 and scale
+    // 0, which the server widens to those of the value, so 
lit(Decimal("1.5")) is a
+    // decimal(10,1).
     if let Ok(dec_mod) = py.import("decimal") {
         let decimal_cls = dec_mod.getattr("Decimal")?;
         if obj.is_instance(&decimal_cls)? {
-            let value: String = obj.str()?.extract()?;
-            let as_tuple = obj.call_method0("as_tuple")?;
-            let num_digits = as_tuple.get_item(1)?.len()? as i32;
-            let exp: i64 = as_tuple.get_item(2)?.extract().unwrap_or(0);
-            let scale = if exp < 0 { (-exp) as i32 } else { 0 };
-            let precision = num_digits.max(scale).max(1);
             return Ok(Some(LiteralExpression::Decimal {
-                value,
-                precision,
-                scale,
+                value: obj.str()?.extract()?,
+                precision: 10,
+                scale: 0,
             }));
         }
     }
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e2adc58..e1e380a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -824,6 +824,27 @@ def 
test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]
 
 
+def test_decimal_literals_are_of_the_types_of_pyspark(session):
+    # The types of official PySpark 4.3.0: at least the precision 10 of 
DecimalType().
+    from decimal import Decimal
+
+    df = session.range(1).select(
+        F.lit(Decimal("1.5")).alias("a"),
+        F.lit(Decimal(0)).alias("b"),
+        F.lit(Decimal("123456789012.5")).alias("c"),
+        F.lit(Decimal("-0.001")).alias("d"),
+        (F.lit(Decimal("1.5")) * F.lit(Decimal("2.25"))).alias("product"),
+    )
+    assert df.dtypes == [
+        ("a", "decimal(10,1)"), ("b", "decimal(10,0)"), ("c", "decimal(13,1)"),
+        ("d", "decimal(10,3)"), ("product", "decimal(21,3)"),
+    ]
+    assert tuple(df.first()) == (
+        Decimal("1.5"), Decimal("0"), Decimal("123456789012.5"), 
Decimal("-0.001"),
+        Decimal("3.375"),
+    )
+
+
 def test_ml_vector_functions_as_pyspark(session):
     from pyspark.ml.functions import array_to_vector, vector_to_array
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index ab23102..b3733b3 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -634,6 +634,33 @@ def test_show_checks_its_arguments_as_pyspark():
         df._show_string(1, n=2)
 
 
+def 
test_decimal_literal_is_of_the_default_decimal_type_as_pyspark(mock_connect):
+    # As PySpark's LiteralExpression of a Decimal, of DecimalType(): precision 
10 and scale
+    # 0, which the server widens to those of the value, wherever the literal 
is.
+    from decimal import Decimal
+
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    spark.range(1).select(
+        F.lit(Decimal("1.5")), F.col("id") == Decimal("-0.001"), 
F.lit(Decimal("12345678901.25"))
+    ).collect()
+    # ExecutePlanRequest.plan=3, Plan.root=1, Relation.project=3, 
Project.expressions=3;
+    # Expression.literal=1, Literal.decimal=12 (value=1, precision=2, scale=3);
+    # .unresolved_function=3, UnresolvedFunction.arguments=2.
+    plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+    expressions = parse_message(parse_message(plan)[3][0])[3]
+
+    def decimal(literal):
+        return parse_message(parse_message(literal)[12][0])
+
+    first, compared, wide = (parse_message(e) for e in expressions)
+    compared = parse_message(parse_message(compared[3][0])[2][1])
+    assert decimal(first[1][0]) == {1: [b"1.5"], 2: [10], 3: [0]}
+    assert decimal(compared[1][0]) == {1: [b"-0.001"], 2: [10], 3: [0]}
+    assert decimal(wide[1][0]) == {1: [b"12345678901.25"], 2: [10], 3: [0]}
+
+
 def test_dropin_module_imports():
     # The thin re-export shims import cleanly.
     import pyspark.sql.dataframe  # noqa: F401


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to