This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a9b8ace  [SPARK-60949][CONNECT][PYTHON] Access struct fields as 
attributes of a Column, and slice a Column, as PySpark does
a9b8ace is described below

commit a9b8acef99ffa7067c5a6207b079aa768be754ea
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 13:06:13 2026 +0900

    [SPARK-60949][CONNECT][PYTHON] Access struct fields as attributes of a 
Column, and slice a Column, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60949](https://issues.apache.org/jira/browse/SPARK-60949): Column 
attributes and slices now work as in PySpark 4.3.0.
    
    - **Attributes:** an attribute that a Column does not have is the field of 
that name, as `Column.__getattr__` gives it. So `df.s.x`, `F.col("s").y.z` and 
`F.col("a")._private` extract fields. As in PySpark:
      - `_jc` raises `PySparkAttributeError` `JVM_ATTRIBUTE_NOT_SUPPORTED`;
      - a name that starts with `__` raises `ATTRIBUTE_NOT_SUPPORTED`, so 
`hasattr(col, "__array__")` is still false.
    - **Slices:** a slice of a Column is `substr(start, stop)`, as 
`Column.__getitem__` gives it: `F.col("t")[2:3]` is `substr(t, 2, 3)`. A slice 
with a step raises `PySparkValueError` `SLICE_WITH_STEP`.
    - **`Column.substr`** checks its arguments as PySpark does. Arguments of 
different types raise `PySparkTypeError` `NOT_SAME_TYPE`, such as `substr(1, 
F.lit(2))` or the open slice `[:3]`. Arguments other than a Column or an int 
raise `NOT_EXPECTED_TYPE`. `pyspark.sql.utils` gains PySpark's `enum_to_value`, 
which `substr` applies to its arguments first, as PySpark does.
    
    ### Why are the changes needed?
    
    - Every such attribute raised `AttributeError`, so `df.select(df.s.x)` 
failed.
    - A slice raised `TypeError`.
    - `substr` built a column of any arguments, such as `substr(a, NULL, 3)`, 
where PySpark raises.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these work as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A probe of 21 attribute, item, slice and `substr` cases gives the same 
results and errors as official PySpark 4.3.0. Two kinds of difference remain, 
both in reprs, which 
[SPARK-60948](https://issues.apache.org/jira/browse/SPARK-60948) covers: a 
field renders `a[b]` where PySpark renders `a['b']`, and a bool literal `true` 
where PySpark renders `True`.
    - Against a local Spark 4.3.0 server, selecting nested fields by attribute 
and slices gives the same column names and values as official PySpark 4.3.0.
    - A new offline test checks the attributes, the slices and the errors, and 
a new live test checks the selection. Both also pass with official PySpark 
4.3.0, and the offline test fails without this change.
    - The lib tests and every integration test, e2e included, against the local 
server (643 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (614 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (58 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #186 from HyukjinKwon/SPARK-60949-column-getattr.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/column.rs     | 77 +++++++++++++++++++++++++++++++++++--
 python/pyspark/sql/utils.py         |  6 +++
 python/tests/test_dropin_e2e.py     |  7 ++++
 python/tests/test_dropin_offline.py | 40 +++++++++++++++++++
 4 files changed, 127 insertions(+), 3 deletions(-)

diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 872613b..9472c6d 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -160,8 +160,44 @@ impl PyColumn {
 
     /// Substring extraction (start and length as columns).
     fn substr(&self, startPos: &Bound<'_, PyAny>, length: &Bound<'_, PyAny>) 
-> PyResult<PyColumn> {
-        let start_col = py_obj_to_column(startPos)?;
-        let length_col = py_obj_to_column(length)?;
+        // As PySpark: the arguments, as values of an Enum, must be of one 
type, a Column or
+        // an int.
+        let py = startPos.py();
+        let enum_to_value = 
py.import("pyspark.sql.utils")?.getattr("enum_to_value")?;
+        let start = enum_to_value.call1((startPos,))?;
+        let length = enum_to_value.call1((length,))?;
+        let type_name = |value: &Bound<'_, PyAny>| -> PyResult<String> {
+            Ok(value.get_type().name()?.to_string())
+        };
+        if !start.get_type().is(length.get_type()) {
+            let (start_type, length_type) = (type_name(&start)?, 
type_name(&length)?);
+            return Err(raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "NOT_SAME_TYPE",
+                &[
+                    ("arg_name1", "startPos"),
+                    ("arg_name2", "length"),
+                    ("arg_type1", &start_type),
+                    ("arg_type2", &length_type),
+                ],
+            ));
+        }
+        if !(length.is_instance_of::<PyColumn>() || 
length.is_instance_of::<pyo3::types::PyInt>()) {
+            let length_type = type_name(&length)?;
+            return Err(raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "NOT_EXPECTED_TYPE",
+                &[
+                    ("expected_type", "Column or int"),
+                    ("arg_name", "startPos"),
+                    ("arg_type", &length_type),
+                ],
+            ));
+        }
+        let start_col = py_obj_to_column(&start)?;
+        let length_col = py_obj_to_column(&length)?;
         Ok(PyColumn::new(
             self.column.clone().substr(start_col, length_col),
         ))
@@ -238,12 +274,47 @@ impl PyColumn {
         Ok(PyColumn::new(self.column.clone().otherwise(val_col)))
     }
 
-    /// Get item by key/index.
+    /// Get item by key/index, or a slice as `substr(start, stop)`. Mirrors
+    /// `Column.__getitem__`.
     fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+        if key.is_instance_of::<pyo3::types::PySlice>() {
+            if !key.getattr("step")?.is_none() {
+                return Err(raise_pyspark(
+                    key.py(),
+                    "PySparkValueError",
+                    "SLICE_WITH_STEP",
+                    &[],
+                ));
+            }
+            return self.substr(&key.getattr("start")?, &key.getattr("stop")?);
+        }
         let key_col = py_obj_to_column(key)?;
         Ok(PyColumn::new(self.column.clone().get_item(key_col)))
     }
 
+    /// An attribute that a Column does not have is the field of that name, as
+    /// `col["name"]`. Mirrors `Column.__getattr__`, which raises for `_jc` 
and for a name
+    /// that starts with `__`.
+    fn __getattr__(&self, py: Python<'_>, item: &str) -> PyResult<PyColumn> {
+        if item == "_jc" {
+            return Err(raise_pyspark(
+                py,
+                "PySparkAttributeError",
+                "JVM_ATTRIBUTE_NOT_SUPPORTED",
+                &[("attr_name", "_jc")],
+            ));
+        }
+        if item.starts_with("__") {
+            return Err(raise_pyspark(
+                py,
+                "PySparkAttributeError",
+                "ATTRIBUTE_NOT_SUPPORTED",
+                &[("attr_name", item)],
+            ));
+        }
+        self.__getitem__(pyo3::types::PyString::new(py, item).as_any())
+    }
+
     /// Equality (==) - returns a Column condition
     fn __eq__(&self, py: Python<'_>, other: Py<PyAny>) -> PyResult<PyColumn> {
         let other_col = to_column(&other.bind(py))?;
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 74a47e9..f8cc0bd 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -16,12 +16,18 @@
 #
 """Small subset of pyspark.sql.utils needed by the drop-in and vendored 
modules."""
 
+from enum import Enum
 from typing import Any, Union, TYPE_CHECKING, Sequence
 
 if TYPE_CHECKING:
     from pyspark.pandas._typing import IndexOpsLike, SeriesOrIndex
 
 
+def enum_to_value(value: Any) -> Any:
+    """Convert an Enum to its value if it is not None."""
+    return enum_to_value(value.value) if value is not None and 
isinstance(value, Enum) else value
+
+
 def is_remote() -> bool:
     """This client is always a Spark Connect (remote) client."""
     return True
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ba50415..4a4cd40 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -460,6 +460,13 @@ def test_collected_rows_behave_as_pyspark_rows(session):
     assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
 
 
+def test_column_attributes_select_struct_fields_and_slices_substrings(session):
+    df = session.sql("SELECT named_struct('x', 1, 'y', named_struct('z', 'q')) 
AS s, 'abcdef' AS t")
+    out = df.select(df.s.x, df.s.y.z, F.col("s").x.alias("x2"), df.t[2:3], 
F.col("t")[F.lit(1):F.lit(2)])
+    assert out.columns == ["s.x", "s.y.z", "x2", "substr(t, 2, 3)", "substr(t, 
1, 2)"]
+    assert tuple(out.first()) == (1, "q", 1, "bcd", "ab")
+
+
 def test_sql_runs_statements_eagerly(session):
     # sql() sends the statement at once: a command takes effect without an 
action, and the
     # DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 97f16fe..bb0ffb8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -843,6 +843,46 @@ def test_misc_functions():
     assert is_col(F.call_function("my_fn", "a"))
 
 
+def test_column_attributes_are_fields_and_slices_are_substrings_as_pyspark():
+    from pyspark.errors import PySparkAttributeError, PySparkTypeError, 
PySparkValueError
+    from pyspark.sql import functions as F
+
+    c = F.col("a")
+    # An attribute that a Column does not have is the field of that name.
+    assert repr(c.b) == repr(c["b"])
+    assert repr(c.b.c) == repr(c["b"]["c"])
+    assert repr(c._private) == repr(c["_private"])
+    assert callable(c.alias)
+    for name, condition in (("_jc", "JVM_ATTRIBUTE_NOT_SUPPORTED"), ("__x__", 
"ATTRIBUTE_NOT_SUPPORTED")):
+        with pytest.raises(PySparkAttributeError) as error:
+            getattr(c, name)
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == {"attr_name": name}
+    assert not hasattr(c, "__array__")
+    # A slice is substr(start, stop), whose arguments are of one type, a 
Column or an int.
+    assert repr(c[1:3]) == "Column<'substr(a, 1, 3)'>"
+    assert repr(c[F.lit(1):F.lit(2)]) == "Column<'substr(a, 1, 2)'>"
+    with pytest.raises(PySparkValueError) as error:
+        c[1:3:2]
+    assert error.value.getCondition() == "SLICE_WITH_STEP"
+    for args, condition, parameters in (
+        ((1, F.lit(2)), "NOT_SAME_TYPE", {
+            "arg_name1": "startPos", "arg_name2": "length", "arg_type1": "int",
+            "arg_type2": "Column"}),
+        ((None, 3), "NOT_SAME_TYPE", {
+            "arg_name1": "startPos", "arg_name2": "length", "arg_type1": 
"NoneType",
+            "arg_type2": "int"}),
+        (("x", "y"), "NOT_EXPECTED_TYPE", {
+            "expected_type": "Column or int", "arg_name": "startPos", 
"arg_type": "str"}),
+    ):
+        with pytest.raises(PySparkTypeError) as error:
+            c.substr(*args)
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+    with pytest.raises(PySparkTypeError, match="NOT_SAME_TYPE"):
+        c[:3]
+
+
 def test_broadcast_type_check():
     from pyspark.sql import functions as F
     from pyspark.errors import PySparkTypeError


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to