This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 78368f4 [SPARK-60697][CONNECT] Avoid DecimalTuple allocation for
exact Decimal inputs
78368f4 is described below
commit 78368f4ed2bd8f10de27ce2d8c88a07c23ffb237
Author: DB Tsai <[email protected]>
AuthorDate: Sat Nov 7 11:21:48 2026 +0900
[SPARK-60697][CONNECT] Avoid DecimalTuple allocation for exact Decimal
inputs
### What changes were proposed in this pull request?
Stacked on #132 (SPARK-60696); until it merges, this diff includes it.
`createDataFrame` no longer builds a `DecimalTuple` (`Decimal.as_tuple()`,
a tuple of every digit) for exact `decimal.Decimal` inputs:
- `decimal_to_value` takes the scale of an exact Decimal from its text,
`str()`, which #132 sends to the encoder anyway (`decimal_string_scale`).
- The type of an inferred pandas column of Decimals, which is taken from
all its values as Arrow does (`Decimal("123.4500")` gives `decimal(7, 4)`),
comes from the same text of each exact Decimal (`decimal_text_digits`), instead
of a second call of `as_tuple()` per cell.
A subclass of `Decimal` can override `__str__` or `as_tuple`, so it is
still asked for both, as Arrow does.
### Why are the changes needed?
[SPARK-60697](https://issues.apache.org/jira/browse/SPARK-60697): the
tuples were built only to read a digit count and an exponent that the text
already has. Measured on macOS against a local Spark 4.3.0 server, as the
client CPU time of `createDataFrame` of a pandas DataFrame of 200,000 rows of
two Decimal columns (median of 5):
| Schema | #132 | This change | PySpark 4.3.0 |
| --- | --- | --- | --- |
| Column names (types inferred) | 1,183 ms | 646 ms | 626 ms |
| Explicit `decimal(12,4), decimal(10,2)` | 840 ms | 630 ms | 205 ms |
All give the same types and the same sums of both columns.
### Does this PR introduce _any_ user-facing change?
No, other than speed.
### How was this patch tested?
- Offline tests in `python/tests/test_dropin_offline.py`, against #132's
mock server that records the uploaded Arrow IPC: for local rows and pandas,
with an explicit schema, column names or none, and fixed-point and scientific
values, the IPC holds the cells and types of PySpark (an inferred pandas column
typed by its values as `pa.array(values).type`, a local one `decimal(38, 18)`),
whatever the Decimal context of the thread; subclasses are still asked for
`__str__` and `as_tuple`, a [...]
- Against the current master, with #132, and a local Spark 4.3.0 server:
the Decimal tests of the offline suite (77 passed, 1 skipped); the full suites
run before the merge.
Changes on top of the original commits: the merge with master and #132's
rework; inferring pandas Decimal columns from the text too, as master now types
them by their values; and tests that expect PySpark's inferred types and share
#132's fixture.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the rework on the current master by
Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #133 from dbtsai/codex/spark-60697-decimal-tuple.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 96 ++++++++++++++++++++++++-------
python/tests/test_dropin_offline.py | 109 ++++++++++++++++++++++++++++++++++++
2 files changed, 185 insertions(+), 20 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index d17c8c3..a5d8cdd 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -652,7 +652,7 @@ impl PySparkSession {
.filter_map(|row| row.get(i))
.find(|value| !matches!(value, Value::Null));
let data_type = match first {
- Some(Value::Decimal { .. }) =>
pandas_decimal_type(&data, i)?,
+ Some(Value::Decimal { .. }) =>
pandas_decimal_type(&data, &rows, i)?,
Some(value) => value_to_datatype(value),
None =>
pandas_dtype_type(dtypes.get(i).map_or("object", String::as_str)),
};
@@ -1687,21 +1687,37 @@ fn pandas_dtype_type(dtype: &str) -> DataType {
/// The decimal type of column `i` of pandas rows, as Arrow infers it from all
its Decimals:
/// the most integer digits and the largest scale of the values.
-fn pandas_decimal_type(rows: &Bound<'_, PyList>, i: usize) ->
PyResult<DataType> {
+fn pandas_decimal_type(data: &Bound<'_, PyList>, rows: &[Row], i: usize) ->
PyResult<DataType> {
+ // The digits and exponent of an exact Decimal come from the text that
`py_to_value` took
+ // of it, rather than from `as_tuple()`, which builds a DecimalTuple of
every digit; a
+ // subclass, which can override either, is asked for `as_tuple()`, as
Arrow does.
+ let decimal_cls = data.py().import("decimal")?.getattr("Decimal")?;
let (mut integer_digits, mut scale) = (0i64, 0i64);
- for row in rows.iter() {
+ for (row, converted) in data.iter().zip(rows) {
let value = row.get_item(i)?;
if value.is_none() {
continue;
}
- let parts = value.call_method0("as_tuple")?;
- let digits = parts.getattr("digits")?.len()? as i64;
- // NaN and infinities have no exponent of digits, which Arrow cannot
type either.
- let Ok(exponent) = parts.getattr("exponent")?.extract::<i64>() else {
- return Err(pyo3::exceptions::PyValueError::new_err(format!(
- "Decimal {} has no precision to type a column by",
- value.str()?
- )));
+ let from_text = match converted.get(i) {
+ Some(Value::Decimal { value: text, .. }) if
value.get_type().is(&decimal_cls) => {
+ decimal_text_digits(text)
+ }
+ _ => None,
+ };
+ let (digits, exponent) = match from_text {
+ Some(parts) => parts,
+ None => {
+ let parts = value.call_method0("as_tuple")?;
+ let digits = parts.getattr("digits")?.len()? as i64;
+ // NaN and infinities have no exponent of digits, which Arrow
cannot type either.
+ let Ok(exponent) = parts.getattr("exponent")?.extract::<i64>()
else {
+ return Err(pyo3::exceptions::PyValueError::new_err(format!(
+ "Decimal {} has no precision to type a column by",
+ value.str()?
+ )));
+ };
+ (digits, exponent)
+ }
};
let (precision, value_scale) = if exponent < 0 {
(digits.max(-exponent), -exponent)
@@ -2050,7 +2066,7 @@ pub(crate) fn datetime_to_micros(obj: &Bound<'_, PyAny>)
-> PyResult<i64> {
}
/// If `obj` is a `decimal.Decimal`, return it as a `Value::Decimal` (string
value + scale
-/// from `as_tuple()`); otherwise `None`. Checked before int/float in
`py_to_value` because
+/// from its exponent); otherwise `None`. Checked before int/float in
`py_to_value` because
/// Decimal defines `__float__`/`__int__`.
fn decimal_to_value(obj: &Bound<'_, PyAny>) -> PyResult<Option<Value>> {
let py = obj.py();
@@ -2066,20 +2082,60 @@ fn decimal_to_value(obj: &Bound<'_, PyAny>) ->
PyResult<Option<Value>> {
// either notation, in work bounded by the length of the text, which a
fixed-point form
// of `1E+1000000000` would not be.
let dec_str: String = obj.str()?.extract()?;
- let as_tuple = obj.call_method0("as_tuple")?;
- let mut scale: Option<i32> = None;
- if let Ok(exp_obj) = as_tuple.get_item(2) {
- if !exp_obj.is_none() {
- if let Ok(exp) = exp_obj.extract::<i64>() {
- if exp <= 0 {
- scale = Some((-exp) as i32);
+ let scale = if obj.get_type().is(&decimal_cls) {
+ // Exact Decimal strings preserve the coefficient's trailing zeros and
exponent.
+ // Subclasses may override either __str__ or as_tuple, so keep their
old path.
+ decimal_string_scale(&dec_str)
+ } else {
+ let as_tuple = obj.call_method0("as_tuple")?;
+ let mut scale = None;
+ if let Ok(exp_obj) = as_tuple.get_item(2) {
+ if !exp_obj.is_none() {
+ if let Ok(exp) = exp_obj.extract::<i64>() {
+ if exp <= 0 {
+ scale = Some((-exp) as i32);
+ }
}
}
}
- }
+ scale
+ };
Ok(Some(Value::Decimal {
value: dec_str,
precision: None,
scale,
}))
}
+
+/// The scale of `Decimal.as_tuple()`, from the text of an exact Decimal: the
negated exponent,
+/// or `None` for a positive exponent and for NaN and the infinities.
+fn decimal_string_scale(value: &str) -> Option<i32> {
+ let (_, exponent) = decimal_text_digits(value)?;
+ // As the scale of `as_tuple()`, including its i32 cast.
+ (exponent <= 0).then(|| (-exponent) as i32)
+}
+
+/// The number of digits and the exponent of `Decimal.as_tuple()`, from the
text (`str()`) of
+/// an exact Decimal, which keeps every digit of its coefficient and its
exponent: `123.4500`
+/// has 7 digits and the exponent -4, `1E+3` 1 and 3, `0.000` 1 and -3. `None`
for NaN and the
+/// infinities, which have no such exponent.
+fn decimal_text_digits(value: &str) -> Option<(i64, i64)> {
+ let unsigned = value.trim_start_matches(['-', '+']);
+ if !unsigned.as_bytes().first()?.is_ascii_digit() {
+ return None;
+ }
+ let (coefficient, exponent) = match unsigned.split_once(['E', 'e']) {
+ Some((coefficient, exponent)) => (coefficient,
exponent.parse::<i64>().ok()?),
+ None => (unsigned, 0),
+ };
+ let (integer, fraction) =
coefficient.split_once('.').unwrap_or((coefficient, ""));
+ // The digits from the first nonzero one; zero has the single digit 0.
+ let leading_zeros = integer.len() - integer.trim_start_matches('0').len();
+ let leading_zeros = if leading_zeros == integer.len() {
+ leading_zeros + (fraction.len() -
fraction.trim_start_matches('0').len())
+ } else {
+ leading_zeros
+ };
+ let digits = (integer.len() + fraction.len() - leading_zeros).max(1) as
i64;
+ Some((digits, exponent - fraction.len() as i64))
+}
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index f9fc331..8db56e2 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6162,6 +6162,115 @@ def test_decimal_rescaling_follows_pyspark(
assert not payload.exists()
[email protected]("kind", ["tuples", "pandas"])
[email protected]("schema_mode", ["explicit", "names", "inferred"])
[email protected]("texts", [
+ ["123.4500", "-2.3400", "0.0000", "-0.0000", None],
+ ["1.2300E+2", "-2.3400E+2", "0.00", "-0.00", None],
+ ["0.001234500", "-0.000002300", "0.000000", None],
+ ["123", "-4", "0", None],
+ ["1E+3", "1E-7", "-0E-7", "1.2300E-7", None],
+])
+def test_decimal_scale_input_ipc(decimal_ipc_session, kind, schema_mode,
texts):
+ # The scale of a Decimal comes from its text, not from as_tuple(): the IPC
holds the same
+ # cells and types as PySpark's, which types an inferred pandas column by
its values as
+ # Arrow does, and a local one as decimal(38, 18). The Decimal context of
the thread,
+ # here one that would round and trap, changes nothing.
+ from decimal import Decimal, Inexact, ROUND_UP, localcontext
+ import pyarrow as pa
+
+ spark, payload = decimal_ipc_session
+ values = [None if text is None else Decimal(text) for text in texts]
+ name = "_1" if schema_mode == "inferred" and kind == "tuples" else "v"
+ data = [(value,) for value in values]
+ if kind == "pandas":
+ pd = pytest.importorskip("pandas")
+ data = pd.DataFrame({name: values}, dtype=object)
+ if schema_mode == "explicit":
+ decimal_type = pa.decimal128(18, 9)
+ schema = T.StructType([T.StructField(name, T.DecimalType(18, 9))])
+ else:
+ decimal_type = pa.decimal128(38, 18) if kind == "tuples" else
pa.array(values).type
+ schema = [name] if schema_mode == "names" else None
+ with localcontext() as context:
+ context.prec = 2
+ context.rounding = ROUND_UP
+ context.traps[Inexact] = True
+ spark.createDataFrame(data, schema).collect()
+ table = pa.ipc.open_stream(payload.read_bytes()).read_all()
+ expected = pa.table({name: pa.array(values, type=decimal_type)})
+ table.validate(full=True)
+ assert table.schema == expected.schema
+ assert table.equals(expected)
+
+
+def test_decimal_scale_of_subclasses(decimal_ipc_session):
+ # A subclass can override __str__ and as_tuple, so it is still asked for
both, as Arrow
+ # does; their errors are raised.
+ from decimal import Decimal
+ import pyarrow as pa
+
+ pd = pytest.importorskip("pandas")
+ spark, payload = decimal_ipc_session
+ calls = []
+
+ class Traced(Decimal):
+ def __str__(self):
+ calls.append("str")
+ return super().__str__()
+
+ def as_tuple(self):
+ calls.append("tuple")
+ return super().as_tuple()
+
+ values = [Traced("1.20"), None, Traced("-30.5")]
+ spark.createDataFrame(pd.DataFrame({"v": values}, dtype=object),
["v"]).collect()
+ assert calls.count("str") == 2 and "tuple" in calls
+ table = pa.ipc.open_stream(payload.read_bytes()).read_all()
+ expected = pa.array([Decimal("1.20"), None, Decimal("-30.5")])
+ assert table.column(0).combine_chunks().equals(expected)
+
+ for hook in ["__str__", "as_tuple"]:
+ def fail(self):
+ raise LookupError("decimal hook")
+
+ failing = type("Failing", (Decimal,), {hook: fail})
+ with pytest.raises(LookupError, match="decimal hook"):
+ spark.createDataFrame(pd.DataFrame({"v": [failing("1.5")]},
dtype=object), ["v"])
+
+
[email protected]("schema", [["v"], T.StructType([T.StructField("v",
T.DecimalType(10, 4))])])
+def test_decimal_scale_avoids_coefficient_tuple(decimal_ipc_session, schema):
+ # Exact Decimals are typed and converted from their text: no DecimalTuple
is built.
+ import cProfile
+ from decimal import Decimal
+
+ pd = pytest.importorskip("pandas")
+ spark, _ = decimal_ipc_session
+ tuple_constructor = type(Decimal("1.20").as_tuple()).__new__.__code__
+ values = pd.DataFrame({"v": [Decimal("123.4500"), Decimal("-2.3000"),
None]}, dtype=object)
+ profile = cProfile.Profile()
+ profile.enable()
+ try:
+ spark.createDataFrame(values, schema)
+ finally:
+ profile.disable()
+ assert not any(entry.code is tuple_constructor for entry in
profile.getstats())
+
+
[email protected]("text", ["Infinity", "-Infinity"])
+def test_decimal_scale_of_non_finite_values(decimal_ipc_session, text):
+ # An infinity has no precision to type a column by.
+ from decimal import Decimal
+
+ pd = pytest.importorskip("pandas")
+ spark, payload = decimal_ipc_session
+ data = pd.DataFrame({"v": [Decimal(text), Decimal("1.5")]}, dtype=object)
+ with pytest.raises(ValueError, match="has no precision to type a column
by"):
+ spark.createDataFrame(data, ["v"])
+ assert not payload.exists()
+
+
# ---------------------------------------------------------------------------
schema cache
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]