This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9bbbb48  [SPARK-61123][CONNECT][PYTHON] Take a Python Enum as the 
literal of its value in lit, Column operators and functions, in the Python 
client as PySpark does
9bbbb48 is described below

commit 9bbbb4816f87000b9502d03f98f855ffb3e42738
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 06:23:55 2026 +0900

    [SPARK-61123][CONNECT][PYTHON] Take a Python Enum as the literal of its 
value in lit, Column operators and functions, in the Python client as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61123](https://issues.apache.org/jira/browse/SPARK-61123): A Python 
`Enum` is the literal of its value, as PySpark's `enum_to_value` makes it, in 
`lit`, the Column operators and the functions that take a value. An `Enum` of 
an `Enum` unwraps fully.
    
    The conversion of a Python value to a literal now takes the value of an 
`Enum`. `lit`, the operators (`+`, `==`, `&`, `**`, ...), `isin`, `between`, 
`contains`, `like`, `bitwiseOR`, `F.pow`, `F.concat_ws`, `F.array_contains` and 
the other functions that take a value all use it.
    
    ### Why are the changes needed?
    
    The client raised `PySparkTypeError: [UNSUPPORTED_DATA_TYPE] Unsupported 
DataType 'IntEnum'` for all of them except `substr`, while PySpark takes an 
`Enum` as its value. Apache Spark's `test_enum_literals` checks them.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `Enum` values work as literals, as with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks the expressions of `Enum` operands of `lit`, 
the operators and functions, and an `Enum` of an `Enum`. It passes with 
official PySpark 4.3.0.
    - 22 cases give the same expressions as official PySpark 4.3.0.
    - Apache Spark's `test_enum_literals` of `test_parity_column.py`, both 
variants, pass on this client against a local server.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2168 passed, 8 skipped), and the 
live drop-in and Arrow suites (173 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #318 from HyukjinKwon/SPARK-61123-enum-literals.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs  | 18 ++++++++++++++++++
 python/tests/test_dropin_offline.py | 38 +++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index 3a2c0ed..16c527f 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -189,12 +189,30 @@ pub fn register_functions(_py: Python<'_>, m: &Bound<'_, 
PyModule>) -> PyResult<
     Ok(())
 }
 
+/// `enum.Enum`, of whose instances a literal is of the value.
+static ENUM: pyo3::sync::PyOnceLock<Py<pyo3::types::PyType>> = 
pyo3::sync::PyOnceLock::new();
+
+/// The value of `obj`, the value of the value of an `Enum` while it is one, 
else `obj`, as
+/// PySpark's `pyspark.sql.utils.enum_to_value`, which its `lit`, Column 
operators and functions
+/// apply to the values that they take.
+pub(crate) fn enum_to_value<'py>(obj: &Bound<'py, PyAny>) -> 
PyResult<Bound<'py, PyAny>> {
+    let py = obj.py();
+    let enum_type = ENUM.import(py, "enum", "Enum")?;
+    let mut value = obj.clone();
+    while value.is_instance(enum_type)? {
+        value = value.getattr("value")?;
+    }
+    Ok(value)
+}
+
 /// Convert a Python object to a Rust Column (handles Column or scalars that 
become literals).
 pub fn to_column(obj: &Bound<'_, PyAny>) -> PyResult<Column> {
     // Try to get as PyColumn first
     if let Ok(col) = obj.extract::<Bound<'_, PyColumn>>() {
         return Ok(col.borrow().column.clone());
     }
+    // An Enum is the literal of its value, as PySpark's.
+    let obj = &enum_to_value(obj)?;
 
     // Check for None
     if obj.is_none() {
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 514a859..2609f54 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1263,6 +1263,44 @@ def 
test_progress_handlers_are_called_with_the_progress_of_executions_as_pyspark
 
 
 
+def test_enums_are_literals_of_their_values_as_pyspark():
+    # As PySpark's enum_to_value: an Enum is the literal of its value, of lit, 
the Column
+    # operators and the functions that take a value, also of an Enum of an 
Enum.
+    from enum import Enum
+
+    from pyspark.sql import functions as F
+
+    class IntEnum(Enum):
+        X = 1
+
+    class StrEnum(Enum):
+        S = "s"
+
+    class BoolEnum(Enum):
+        T = True
+
+    class Nested(Enum):
+        N = IntEnum.X
+
+    c = F.col("id")
+    for column, expected in [
+        (c + IntEnum.X, "+(id, 1)"),
+        (IntEnum.X + c, "+(1, id)"),
+        (c == IntEnum.X, "==(id, 1)"),
+        (F.col("b") & BoolEnum.T, "and(b, True)"),
+        (F.lit(IntEnum.X), "1"),
+        (F.lit(StrEnum.S), "s"),
+        (F.lit(Nested.N), "1"),
+        (c.isin(IntEnum.X), "in(id, 1)"),
+        (c.between(IntEnum.X, 5), "and(>=(id, 1), <=(id, 5))"),
+        (F.col("s").contains(StrEnum.S), "contains(s, s)"),
+        (c ** IntEnum.X, "power(id, 1)"),
+        (F.concat_ws(StrEnum.S, c), "concat_ws(s, id)"),
+        (F.array_contains(F.col("a"), IntEnum.X), "array_contains(a, 1)"),
+    ]:
+        assert repr(column) == f"Column<'{expected}'>"
+
+
 def test_udfs_are_created_as_pyspark(mock_connect):
     # As PySpark's _create_py_udf: udf() without useArrow makes an 
Arrow-optimized UDF
     # (SQL_ARROW_BATCHED_UDF) when the active session enables


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to