This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 10a725c  [SPARK-60680][CONNECT] Support integer positional DataFrame 
column indexing
10a725c is described below

commit 10a725c50654e7d8f93e05902021a34804e2226b
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:35:26 2026 +0900

    [SPARK-60680][CONNECT] Support integer positional DataFrame column indexing
    
    ### What changes were proposed in this pull request?
    
    `DataFrame.__getitem__` now takes what PySpark Connect's takes:
    
    | `df[item]` | PySpark 4.3.0 and now | Before |
    | --- | --- | --- |
    | a `str` | the column of that name | the same |
    | an `int`, `bool` included | the column at that position, with Python's 
list indexing: negative positions, and `IndexError` out of range | `TypeError` |
    | a `Column` | `df.filter(item)` | `TypeError` |
    | a `list` or `tuple` | `df.select(*item)` | `TypeError` |
    | anything else | `PySparkTypeError` `NOT_EXPECTED_TYPE` (`expected_type`: 
"Column, int, list, str or tuple") | PyO3's `TypeError` |
    
    A position looks the column names up through the DataFrame's cached schema; 
the other forms send no RPC.
    
    ### Why are the changes needed?
    
    [SPARK-60680](https://issues.apache.org/jira/browse/SPARK-60680): `df[0]`, 
`df[df.a > 1]` and `df[["a", "b"]]` work in PySpark but raised a `TypeError` 
here.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `df[...]` takes positions, Columns, lists and tuples, and raises 
PySpark's error for other types, as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests against a mock gRPC server check positions (negative 
ones included) over empty, duplicate and dotted names, `IndexError`, subclasses 
of `int` and `str`, `NOT_EXPECTED_TYPE` for floats, `None`, bytes, slices and 
objects with `__index__` without a schema lookup, that Columns, lists and 
tuples give DataFrames without one, and the errors of a failed schema lookup.
    - A new live test checks the rows and columns of each form. It also passes 
with official PySpark 4.3.0.
    - Against the current master, merged into this branch, and a local Spark 
4.3.0 server: the lib and every integration test of `apache-spark-connect` and 
`apache-spark-connect-core`, e2e included (771 passed), the build of all tests 
with `wasm-udf`, the offline Python suites (740 passed, 9 skipped), the live 
drop-in and Arrow suites (77 passed) and the live cases (10 passed). `cargo fmt 
--all --check` passes.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6); the merge with master and the Column, 
list and tuple forms by Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #125 from dbtsai/codex/spark-60680-positional-index.
    
    Lead-authored-by: DB Tsai <[email protected]>
    Co-authored-by: Hyukjin Kwon <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  |  40 ++++++-
 python/tests/test_dropin_e2e.py     |  16 +++
 python/tests/test_dropin_offline.py | 223 ++++++++++++++++++++++++++++++++++++
 3 files changed, 275 insertions(+), 4 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index d41989d..d2a1e11 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1,7 +1,7 @@
 //! PyO3 wrapper for spark_connect::dataframe::DataFrame.
 
 use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyTuple};
+use pyo3::types::{PyDict, PyInt, PyList, PyTuple};
 use spark_connect::dataframe::{DataFrame, LocalRowIterator};
 use spark_connect::plan::JoinType;
 use spark_connect::udf::{eval_type, CommonInlineUserDefinedFunctionExpression, 
PythonUDFPayload};
@@ -1577,9 +1577,41 @@ impl PyDataFrame {
         }
     }
 
-    /// Column access by name (`df["col"]`).
-    fn __getitem__(&self, item: &str) -> PyColumn {
-        PyColumn::new(spark_connect::functions::col(item))
+    /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`) 
or position
+    /// (`df[0]`), the rows that match a Column (`df[df.a > 1]`, as `filter`), 
or the
+    /// projection of a list or tuple of columns (`df[["a", "b"]]`, as 
`select`).
+    fn __getitem__(&self, py: Python<'_>, item: &Bound<'_, PyAny>) -> 
PyResult<Py<PyAny>> {
+        use pyo3::IntoPyObjectExt;
+
+        let name = if let Ok(name) = item.cast::<pyo3::types::PyString>() {
+            name.extract::<String>()?
+        } else if item.is_instance_of::<PyColumn>() {
+            return self.filter(item)?.into_py_any(py);
+        } else if item.is_instance_of::<PyList>() || 
item.is_instance_of::<PyTuple>() {
+            let cols = item.try_iter()?.collect::<PyResult<Vec<_>>>()?;
+            return self.select(py, cols)?.into_py_any(py);
+        } else if item.is_instance_of::<PyInt>() {
+            let columns = self.columns(py)?;
+            // Use Python list indexing for negative indices, bools (an int
+            // subclass), and IndexError even for integers larger than isize.
+            PyList::new(py, columns)?
+                .as_any()
+                .get_item(item)?
+                .extract::<String>()?
+        } else {
+            let arg_type = item.get_type().name()?.to_string();
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "NOT_EXPECTED_TYPE",
+                &[
+                    ("expected_type", "Column, int, list, str or tuple"),
+                    ("arg_name", "item"),
+                    ("arg_type", &arg_type),
+                ],
+            ));
+        };
+        PyColumn::new(spark_connect::functions::col(&name)).into_py_any(py)
     }
 
     /// Private column accessor used by pandas-on-Spark (`sdf._col(name)`); 
mirrors the
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e468224..b694b18 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -667,6 +667,22 @@ def 
test_collected_rows_keep_their_values_as_pyspark(session):
     assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
 
 
+def test_getitem_indexes_filters_and_selects_as_pyspark(session):
+    from pyspark.errors import PySparkTypeError
+
+    df = session.createDataFrame([(1, "a"), (2, "b"), (3, "c")], ["id", "s"])
+    assert df.select(df[0], df[-1], df[True]).columns == ["id", "s", "s"]
+    assert df[df["id"] > 1].collect() == [Row(id=2, s="b"), Row(id=3, s="c")]
+    assert df[["s"]].collect() == [Row(s="a"), Row(s="b"), Row(s="c")]
+    assert df[("s", "id")].columns == ["s", "id"]
+    assert df[[["id"]]].columns == ["id"]
+    with pytest.raises(IndexError):
+        df[2]
+    with pytest.raises(PySparkTypeError) as error:
+        df[1.5]
+    assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+
+
 def test_sql_runs_statements_eagerly(session):
     # sql() sends the statement at once: a command takes effect without an 
action, and the
     # DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index e719248..eaf0602 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5928,6 +5928,229 @@ def 
test_connect_registry_sends_auto_cdc_define_flow(mock_connect):
     assert 10 not in details and 11 not in details and 12 not in details
 
 
+# Positional DataFrame indexing via a separate-process mock gRPC service.
+from concurrent.futures import ThreadPoolExecutor as 
positional_indexing_ThreadPoolExecutor
+import multiprocessing as positional_indexing_multiprocessing
+
+from pyspark.sql import (
+    SparkSession as positional_indexing_SparkSession, Column as 
positional_indexing_Column,
+)
+
+
+def positional_indexing_varint(value):
+    result = bytearray()
+    while value > 127:
+        result.append((value & 127) | 128)
+        value >>= 7
+    result.append(value)
+    return bytes(result)
+
+
+def positional_indexing_message(field, value):
+    if isinstance(value, str):
+        value = value.encode()
+    return positional_indexing_varint(field * 8 + 2) + 
positional_indexing_varint(len(value)) + value
+
+
+def positional_indexing_schema(name="id", nested=False):
+    # types.proto: DataType.long=7, .struct=21; Struct.fields=1;
+    # StructField: name=1, data_type=2, nullable=3, metadata=4.
+    data_type = positional_indexing_message(21, positional_indexing_message(1, 
positional_indexing_field("inner", positional_indexing_message(7, b"")))) if 
nested else positional_indexing_message(7, b"")
+    return positional_indexing_message(21, positional_indexing_message(1, 
positional_indexing_field(name, data_type)))
+
+
+def positional_indexing_field(name, data_type):
+    return (positional_indexing_message(1, name) + 
positional_indexing_message(2, data_type) + b"\x18\x01"
+            + positional_indexing_message(4, '{"note":"original"}'))
+
+
+def positional_indexing_serve(connection):
+    import grpc as positional_indexing_grpc
+    # A separate process also exercises columns/dtypes, whose first access may
+    # hold the client's GIL while waiting for the RPC.
+    class Service:
+        calls = 0
+        data_type = positional_indexing_schema()
+        failure = None
+
+        def analyze(self, request, context):
+            self.calls += 1
+            # Both clients serialize the UUID session identifier as field 1.
+            # Echo it so reference Python can verify response integrity.
+            assert request[0] == 10 and request[1] < 128
+            session = request[:2 + request[1]]
+            if self.failure == "rpc":
+                
context.abort(positional_indexing_grpc.StatusCode.INVALID_ARGUMENT, "schema 
unavailable")
+            if self.failure == "no-result":
+                return b""
+            if self.failure == "missing-schema":
+                return positional_indexing_message(2, b"")
+            if self.failure == "invalid-type":
+                return positional_indexing_message(2, 
positional_indexing_message(1, b""))
+            # base.proto: AnalyzePlanResponse.schema=2, Schema.schema=1.
+            return session + positional_indexing_message(2, 
positional_indexing_message(1, self.data_type))
+
+    service = Service()
+    with positional_indexing_ThreadPoolExecutor(max_workers=4) as executor:
+        server = positional_indexing_grpc.server(executor)
+        handler = 
positional_indexing_grpc.unary_unary_rpc_method_handler(service.analyze)
+        release = positional_indexing_grpc.unary_unary_rpc_method_handler(
+            lambda request, context: request[:2 + request[1]])
+        config = positional_indexing_grpc.unary_unary_rpc_method_handler(
+            lambda request, context: request[:2 + request[1]] + 
positional_indexing_message(
+                2, positional_indexing_message(1, 
"spark.python.sql.dataFrameDebugging.enabled")
+                + positional_indexing_message(2, "false")))
+        
server.add_generic_rpc_handlers((positional_indexing_grpc.method_handlers_generic_handler(
+            "spark.connect.SparkConnectService", {
+                "AnalyzePlan": handler, "ReleaseSession": release, "Config": 
config}),))
+        port = server.add_insecure_port("127.0.0.1:0")
+        server.start()
+        connection.send(port)
+        try:
+            while True:
+                command = connection.recv()
+                if command[0] == "stop":
+                    break
+                if command[0] == "get":
+                    connection.send(getattr(service, command[1]))
+                elif command[0] == "set":
+                    setattr(service, command[1], command[2])
+                    connection.send(None)
+        finally:
+            server.stop(0).wait()
+
+
+class positional_indexing_RemoteService:
+    def __init__(self, connection):
+        object.__setattr__(self, "connection", connection)
+
+    def __getattr__(self, name):
+        self.connection.send(("get", name))
+        assert self.connection.poll(10), "Mock server did not respond"
+        return self.connection.recv()
+
+    def __setattr__(self, name, value):
+        self.connection.send(("set", name, value))
+        assert self.connection.poll(10), "Mock server did not respond"
+        self.connection.recv()
+
+
[email protected]
+def positional_indexing_analyzed():
+    pytest.importorskip("grpc")
+    context = positional_indexing_multiprocessing.get_context("spawn")
+    parent, child = context.Pipe()
+    process = context.Process(target=positional_indexing_serve, args=(child,))
+    process.start()
+    spark = None
+    try:
+        assert parent.poll(20), "Mock server did not start"
+        port = parent.recv()
+        spark = 
positional_indexing_SparkSession.builder.remote(f"sc://127.0.0.1:{port}").create()
+        yield spark, positional_indexing_RemoteService(parent)
+    finally:
+        if spark is not None:
+            spark.stop()
+        if process.is_alive():
+            parent.send(("stop",))
+        process.join(10)
+        if process.is_alive():
+            process.terminate()
+            process.join(10)
+        parent.close()
+        child.close()
+
+
+def positional_indexing_columns(*names):
+    return positional_indexing_message(21, 
b"".join(positional_indexing_message(1, positional_indexing_field(name, 
positional_indexing_message(7, b""))) for name in names))
+
+
[email protected]("index, name", [
+    (0, "first"), (1, "second"), (2, "last"),
+    (-1, "last"), (-2, "second"), (-3, "first"),
+    (False, "first"), (True, "second"),
+])
+def test_positional_indexing_positional_column(positional_indexing_analyzed, 
index, name):
+    spark, service = positional_indexing_analyzed
+    service.data_type = positional_indexing_columns("first", "second", "last")
+    df = spark.range(10)
+    column = df[index]
+    assert isinstance(column, positional_indexing_Column)
+    assert repr(column) == repr(df[name])
+    assert service.calls == 1
+
+
[email protected]("names, index", [
+    (("a", "b"), 2), (("a", "b"), -3),
+    (("a",), 10**100), (("a",), -(10**100)),
+    ((), 0), ((), -1), ((), False), (("a",), True),
+])
+def test_positional_indexing_positional_bounds(positional_indexing_analyzed, 
names, index):
+    spark, service = positional_indexing_analyzed
+    service.data_type = positional_indexing_columns(*names)
+    with pytest.raises(IndexError):
+        spark.range(10)[index]
+    assert service.calls == 1
+
+
+def 
test_positional_indexing_integer_subclass_and_string_lookup(positional_indexing_analyzed):
+    class Position(int):
+        pass
+    class Name(str):
+        pass
+    spark, service = positional_indexing_analyzed
+    service.data_type = positional_indexing_columns("a.b", "with space", "a.b")
+    df = spark.range(10)
+    assert repr(df[Position(-2)]) == repr(df[Name("with space")])
+    assert repr(df[0]) == repr(df["a.b"])
+    before = service.calls
+    for name in ["missing", "*", "`a.b`", ""]:
+        assert isinstance(df[name], positional_indexing_Column)
+    assert service.calls == before
+
+
+def 
test_positional_indexing_invalid_index_types_do_not_analyze(positional_indexing_analyzed):
+    from pyspark.errors import PySparkTypeError
+
+    class Position:
+        def __index__(self):
+            return 0
+    spark, service = positional_indexing_analyzed
+    df = spark.range(10)
+    for value in [0.0, None, b"id", slice(0, 1), Position()]:
+        # As PySpark, which takes a Column, an int, a list, a str or a tuple.
+        with pytest.raises(PySparkTypeError) as error:
+            df[value]
+        assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert error.value.getMessageParameters() == {
+            "expected_type": "Column, int, list, str or tuple", "arg_name": 
"item",
+            "arg_type": type(value).__name__}
+    assert service.calls == 0
+
+
+def 
test_positional_indexing_columns_filter_and_sequences_select(positional_indexing_analyzed):
+    from pyspark.sql import DataFrame
+
+    # As PySpark: a Column filters the rows, and a list or tuple selects its 
columns,
+    # without looking the schema up. The live tests check the rows.
+    spark, service = positional_indexing_analyzed
+    df = spark.range(10)
+    for item in [df["id"] > 1, ["id"], ("id", "id"), [["id"]], []]:
+        assert type(df[item]) is DataFrame
+    assert service.calls == 0
+
+
+def 
test_positional_indexing_positional_propagates_schema_failure(positional_indexing_analyzed):
+    spark, service = positional_indexing_analyzed
+    service.failure = "rpc"
+    df = spark.range(10)
+    with pytest.raises(Exception, match="schema unavailable"):
+        df[0]
+    assert service.calls == 1
+    assert isinstance(df["id"], positional_indexing_Column)
+    assert service.calls == 1
+
+
 # ------------------------------------------------- reader/writer in-place 
setters
 
 @pytest.fixture


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to