This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new a9b8ace [SPARK-60949][CONNECT][PYTHON] Access struct fields as
attributes of a Column, and slice a Column, as PySpark does
a9b8ace is described below
commit a9b8acef99ffa7067c5a6207b079aa768be754ea
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 13:06:13 2026 +0900
[SPARK-60949][CONNECT][PYTHON] Access struct fields as attributes of a
Column, and slice a Column, as PySpark does
### What changes were proposed in this pull request?
[SPARK-60949](https://issues.apache.org/jira/browse/SPARK-60949): Column
attributes and slices now work as in PySpark 4.3.0.
- **Attributes:** an attribute that a Column does not have is the field of
that name, as `Column.__getattr__` gives it. So `df.s.x`, `F.col("s").y.z` and
`F.col("a")._private` extract fields. As in PySpark:
- `_jc` raises `PySparkAttributeError` `JVM_ATTRIBUTE_NOT_SUPPORTED`;
- a name that starts with `__` raises `ATTRIBUTE_NOT_SUPPORTED`, so
`hasattr(col, "__array__")` is still false.
- **Slices:** a slice of a Column is `substr(start, stop)`, as
`Column.__getitem__` gives it: `F.col("t")[2:3]` is `substr(t, 2, 3)`. A slice
with a step raises `PySparkValueError` `SLICE_WITH_STEP`.
- **`Column.substr`** checks its arguments as PySpark does. Arguments of
different types raise `PySparkTypeError` `NOT_SAME_TYPE`, such as `substr(1,
F.lit(2))` or the open slice `[:3]`. Arguments other than a Column or an int
raise `NOT_EXPECTED_TYPE`. `pyspark.sql.utils` gains PySpark's `enum_to_value`,
which `substr` applies to its arguments first, as PySpark does.
### Why are the changes needed?
- Every such attribute raised `AttributeError`, so `df.select(df.s.x)`
failed.
- A slice raised `TypeError`.
- `substr` built a column of any arguments, such as `substr(a, NULL, 3)`,
where PySpark raises.
### Does this PR introduce _any_ user-facing change?
Yes, these work as in PySpark 4.3.0.
### How was this patch tested?
- A probe of 21 attribute, item, slice and `substr` cases gives the same
results and errors as official PySpark 4.3.0. Two kinds of difference remain,
both in reprs, which
[SPARK-60948](https://issues.apache.org/jira/browse/SPARK-60948) covers: a
field renders `a[b]` where PySpark renders `a['b']`, and a bool literal `true`
where PySpark renders `True`.
- Against a local Spark 4.3.0 server, selecting nested fields by attribute
and slices gives the same column names and values as official PySpark 4.3.0.
- A new offline test checks the attributes, the slices and the errors, and
a new live test checks the selection. Both also pass with official PySpark
4.3.0, and the offline test fails without this change.
- The lib tests and every integration test, e2e included, against the local
server (643 passed), the build of all tests with `wasm-udf`, the offline Python
suites (614 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (58 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #186 from HyukjinKwon/SPARK-60949-column-getattr.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/column.rs | 77 +++++++++++++++++++++++++++++++++++--
python/pyspark/sql/utils.py | 6 +++
python/tests/test_dropin_e2e.py | 7 ++++
python/tests/test_dropin_offline.py | 40 +++++++++++++++++++
4 files changed, 127 insertions(+), 3 deletions(-)
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 872613b..9472c6d 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -160,8 +160,44 @@ impl PyColumn {
/// Substring extraction (start and length as columns).
fn substr(&self, startPos: &Bound<'_, PyAny>, length: &Bound<'_, PyAny>)
-> PyResult<PyColumn> {
- let start_col = py_obj_to_column(startPos)?;
- let length_col = py_obj_to_column(length)?;
+ // As PySpark: the arguments, as values of an Enum, must be of one
type, a Column or
+ // an int.
+ let py = startPos.py();
+ let enum_to_value =
py.import("pyspark.sql.utils")?.getattr("enum_to_value")?;
+ let start = enum_to_value.call1((startPos,))?;
+ let length = enum_to_value.call1((length,))?;
+ let type_name = |value: &Bound<'_, PyAny>| -> PyResult<String> {
+ Ok(value.get_type().name()?.to_string())
+ };
+ if !start.get_type().is(length.get_type()) {
+ let (start_type, length_type) = (type_name(&start)?,
type_name(&length)?);
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_SAME_TYPE",
+ &[
+ ("arg_name1", "startPos"),
+ ("arg_name2", "length"),
+ ("arg_type1", &start_type),
+ ("arg_type2", &length_type),
+ ],
+ ));
+ }
+ if !(length.is_instance_of::<PyColumn>() ||
length.is_instance_of::<pyo3::types::PyInt>()) {
+ let length_type = type_name(&length)?;
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "Column or int"),
+ ("arg_name", "startPos"),
+ ("arg_type", &length_type),
+ ],
+ ));
+ }
+ let start_col = py_obj_to_column(&start)?;
+ let length_col = py_obj_to_column(&length)?;
Ok(PyColumn::new(
self.column.clone().substr(start_col, length_col),
))
@@ -238,12 +274,47 @@ impl PyColumn {
Ok(PyColumn::new(self.column.clone().otherwise(val_col)))
}
- /// Get item by key/index.
+ /// Get item by key/index, or a slice as `substr(start, stop)`. Mirrors
+ /// `Column.__getitem__`.
fn __getitem__(&self, key: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+ if key.is_instance_of::<pyo3::types::PySlice>() {
+ if !key.getattr("step")?.is_none() {
+ return Err(raise_pyspark(
+ key.py(),
+ "PySparkValueError",
+ "SLICE_WITH_STEP",
+ &[],
+ ));
+ }
+ return self.substr(&key.getattr("start")?, &key.getattr("stop")?);
+ }
let key_col = py_obj_to_column(key)?;
Ok(PyColumn::new(self.column.clone().get_item(key_col)))
}
+ /// An attribute that a Column does not have is the field of that name, as
+ /// `col["name"]`. Mirrors `Column.__getattr__`, which raises for `_jc`
and for a name
+ /// that starts with `__`.
+ fn __getattr__(&self, py: Python<'_>, item: &str) -> PyResult<PyColumn> {
+ if item == "_jc" {
+ return Err(raise_pyspark(
+ py,
+ "PySparkAttributeError",
+ "JVM_ATTRIBUTE_NOT_SUPPORTED",
+ &[("attr_name", "_jc")],
+ ));
+ }
+ if item.starts_with("__") {
+ return Err(raise_pyspark(
+ py,
+ "PySparkAttributeError",
+ "ATTRIBUTE_NOT_SUPPORTED",
+ &[("attr_name", item)],
+ ));
+ }
+ self.__getitem__(pyo3::types::PyString::new(py, item).as_any())
+ }
+
/// Equality (==) - returns a Column condition
fn __eq__(&self, py: Python<'_>, other: Py<PyAny>) -> PyResult<PyColumn> {
let other_col = to_column(&other.bind(py))?;
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 74a47e9..f8cc0bd 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -16,12 +16,18 @@
#
"""Small subset of pyspark.sql.utils needed by the drop-in and vendored
modules."""
+from enum import Enum
from typing import Any, Union, TYPE_CHECKING, Sequence
if TYPE_CHECKING:
from pyspark.pandas._typing import IndexOpsLike, SeriesOrIndex
+def enum_to_value(value: Any) -> Any:
+ """Convert an Enum to its value if it is not None."""
+ return enum_to_value(value.value) if value is not None and
isinstance(value, Enum) else value
+
+
def is_remote() -> bool:
"""This client is always a Spark Connect (remote) client."""
return True
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ba50415..4a4cd40 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -460,6 +460,13 @@ def test_collected_rows_behave_as_pyspark_rows(session):
assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
+def test_column_attributes_select_struct_fields_and_slices_substrings(session):
+ df = session.sql("SELECT named_struct('x', 1, 'y', named_struct('z', 'q'))
AS s, 'abcdef' AS t")
+ out = df.select(df.s.x, df.s.y.z, F.col("s").x.alias("x2"), df.t[2:3],
F.col("t")[F.lit(1):F.lit(2)])
+ assert out.columns == ["s.x", "s.y.z", "x2", "substr(t, 2, 3)", "substr(t,
1, 2)"]
+ assert tuple(out.first()) == (1, "q", 1, "bcd", "ab")
+
+
def test_sql_runs_statements_eagerly(session):
# sql() sends the statement at once: a command takes effect without an
action, and the
# DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 97f16fe..bb0ffb8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -843,6 +843,46 @@ def test_misc_functions():
assert is_col(F.call_function("my_fn", "a"))
+def test_column_attributes_are_fields_and_slices_are_substrings_as_pyspark():
+ from pyspark.errors import PySparkAttributeError, PySparkTypeError,
PySparkValueError
+ from pyspark.sql import functions as F
+
+ c = F.col("a")
+ # An attribute that a Column does not have is the field of that name.
+ assert repr(c.b) == repr(c["b"])
+ assert repr(c.b.c) == repr(c["b"]["c"])
+ assert repr(c._private) == repr(c["_private"])
+ assert callable(c.alias)
+ for name, condition in (("_jc", "JVM_ATTRIBUTE_NOT_SUPPORTED"), ("__x__",
"ATTRIBUTE_NOT_SUPPORTED")):
+ with pytest.raises(PySparkAttributeError) as error:
+ getattr(c, name)
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == {"attr_name": name}
+ assert not hasattr(c, "__array__")
+ # A slice is substr(start, stop), whose arguments are of one type, a
Column or an int.
+ assert repr(c[1:3]) == "Column<'substr(a, 1, 3)'>"
+ assert repr(c[F.lit(1):F.lit(2)]) == "Column<'substr(a, 1, 2)'>"
+ with pytest.raises(PySparkValueError) as error:
+ c[1:3:2]
+ assert error.value.getCondition() == "SLICE_WITH_STEP"
+ for args, condition, parameters in (
+ ((1, F.lit(2)), "NOT_SAME_TYPE", {
+ "arg_name1": "startPos", "arg_name2": "length", "arg_type1": "int",
+ "arg_type2": "Column"}),
+ ((None, 3), "NOT_SAME_TYPE", {
+ "arg_name1": "startPos", "arg_name2": "length", "arg_type1":
"NoneType",
+ "arg_type2": "int"}),
+ (("x", "y"), "NOT_EXPECTED_TYPE", {
+ "expected_type": "Column or int", "arg_name": "startPos",
"arg_type": "str"}),
+ ):
+ with pytest.raises(PySparkTypeError) as error:
+ c.substr(*args)
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+ with pytest.raises(PySparkTypeError, match="NOT_SAME_TYPE"):
+ c[:3]
+
+
def test_broadcast_type_check():
from pyspark.sql import functions as F
from pyspark.errors import PySparkTypeError
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]