This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 10a725c [SPARK-60680][CONNECT] Support integer positional DataFrame
column indexing
10a725c is described below
commit 10a725c50654e7d8f93e05902021a34804e2226b
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:35:26 2026 +0900
[SPARK-60680][CONNECT] Support integer positional DataFrame column indexing
### What changes were proposed in this pull request?
`DataFrame.__getitem__` now takes what PySpark Connect's takes:
| `df[item]` | PySpark 4.3.0 and now | Before |
| --- | --- | --- |
| a `str` | the column of that name | the same |
| an `int`, `bool` included | the column at that position, with Python's
list indexing: negative positions, and `IndexError` out of range | `TypeError` |
| a `Column` | `df.filter(item)` | `TypeError` |
| a `list` or `tuple` | `df.select(*item)` | `TypeError` |
| anything else | `PySparkTypeError` `NOT_EXPECTED_TYPE` (`expected_type`:
"Column, int, list, str or tuple") | PyO3's `TypeError` |
A position looks the column names up through the DataFrame's cached schema;
the other forms send no RPC.
### Why are the changes needed?
[SPARK-60680](https://issues.apache.org/jira/browse/SPARK-60680): `df[0]`,
`df[df.a > 1]` and `df[["a", "b"]]` work in PySpark but raised a `TypeError`
here.
### Does this PR introduce _any_ user-facing change?
Yes, `df[...]` takes positions, Columns, lists and tuples, and raises
PySpark's error for other types, as in PySpark.
### How was this patch tested?
- New offline tests against a mock gRPC server check positions (negative
ones included) over empty, duplicate and dotted names, `IndexError`, subclasses
of `int` and `str`, `NOT_EXPECTED_TYPE` for floats, `None`, bytes, slices and
objects with `__index__` without a schema lookup, that Columns, lists and
tuples give DataFrames without one, and the errors of a failed schema lookup.
- A new live test checks the rows and columns of each form. It also passes
with official PySpark 4.3.0.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (771 passed), the build of all tests
with `wasm-udf`, the offline Python suites (740 passed, 9 skipped), the live
drop-in and Arrow suites (77 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master and the Column,
list and tuple forms by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #125 from dbtsai/codex/spark-60680-positional-index.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 40 ++++++-
python/tests/test_dropin_e2e.py | 16 +++
python/tests/test_dropin_offline.py | 223 ++++++++++++++++++++++++++++++++++++
3 files changed, 275 insertions(+), 4 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index d41989d..d2a1e11 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1,7 +1,7 @@
//! PyO3 wrapper for spark_connect::dataframe::DataFrame.
use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyTuple};
+use pyo3::types::{PyDict, PyInt, PyList, PyTuple};
use spark_connect::dataframe::{DataFrame, LocalRowIterator};
use spark_connect::plan::JoinType;
use spark_connect::udf::{eval_type, CommonInlineUserDefinedFunctionExpression,
PythonUDFPayload};
@@ -1577,9 +1577,41 @@ impl PyDataFrame {
}
}
- /// Column access by name (`df["col"]`).
- fn __getitem__(&self, item: &str) -> PyColumn {
- PyColumn::new(spark_connect::functions::col(item))
+ /// As PySpark's `DataFrame.__getitem__`: a column by name (`df["col"]`)
or position
+ /// (`df[0]`), the rows that match a Column (`df[df.a > 1]`, as `filter`),
or the
+ /// projection of a list or tuple of columns (`df[["a", "b"]]`, as
`select`).
+ fn __getitem__(&self, py: Python<'_>, item: &Bound<'_, PyAny>) ->
PyResult<Py<PyAny>> {
+ use pyo3::IntoPyObjectExt;
+
+ let name = if let Ok(name) = item.cast::<pyo3::types::PyString>() {
+ name.extract::<String>()?
+ } else if item.is_instance_of::<PyColumn>() {
+ return self.filter(item)?.into_py_any(py);
+ } else if item.is_instance_of::<PyList>() ||
item.is_instance_of::<PyTuple>() {
+ let cols = item.try_iter()?.collect::<PyResult<Vec<_>>>()?;
+ return self.select(py, cols)?.into_py_any(py);
+ } else if item.is_instance_of::<PyInt>() {
+ let columns = self.columns(py)?;
+ // Use Python list indexing for negative indices, bools (an int
+ // subclass), and IndexError even for integers larger than isize.
+ PyList::new(py, columns)?
+ .as_any()
+ .get_item(item)?
+ .extract::<String>()?
+ } else {
+ let arg_type = item.get_type().name()?.to_string();
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "Column, int, list, str or tuple"),
+ ("arg_name", "item"),
+ ("arg_type", &arg_type),
+ ],
+ ));
+ };
+ PyColumn::new(spark_connect::functions::col(&name)).into_py_any(py)
}
/// Private column accessor used by pandas-on-Spark (`sdf._col(name)`);
mirrors the
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e468224..b694b18 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -667,6 +667,22 @@ def
test_collected_rows_keep_their_values_as_pyspark(session):
assert (rows[0].v.toJson(), rows[0].n) == ("1", 2)
+def test_getitem_indexes_filters_and_selects_as_pyspark(session):
+ from pyspark.errors import PySparkTypeError
+
+ df = session.createDataFrame([(1, "a"), (2, "b"), (3, "c")], ["id", "s"])
+ assert df.select(df[0], df[-1], df[True]).columns == ["id", "s", "s"]
+ assert df[df["id"] > 1].collect() == [Row(id=2, s="b"), Row(id=3, s="c")]
+ assert df[["s"]].collect() == [Row(s="a"), Row(s="b"), Row(s="c")]
+ assert df[("s", "id")].columns == ["s", "id"]
+ assert df[[["id"]]].columns == ["id"]
+ with pytest.raises(IndexError):
+ df[2]
+ with pytest.raises(PySparkTypeError) as error:
+ df[1.5]
+ assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+
+
def test_sql_runs_statements_eagerly(session):
# sql() sends the statement at once: a command takes effect without an
action, and the
# DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index e719248..eaf0602 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5928,6 +5928,229 @@ def
test_connect_registry_sends_auto_cdc_define_flow(mock_connect):
assert 10 not in details and 11 not in details and 12 not in details
+# Positional DataFrame indexing via a separate-process mock gRPC service.
+from concurrent.futures import ThreadPoolExecutor as
positional_indexing_ThreadPoolExecutor
+import multiprocessing as positional_indexing_multiprocessing
+
+from pyspark.sql import (
+ SparkSession as positional_indexing_SparkSession, Column as
positional_indexing_Column,
+)
+
+
+def positional_indexing_varint(value):
+ result = bytearray()
+ while value > 127:
+ result.append((value & 127) | 128)
+ value >>= 7
+ result.append(value)
+ return bytes(result)
+
+
+def positional_indexing_message(field, value):
+ if isinstance(value, str):
+ value = value.encode()
+ return positional_indexing_varint(field * 8 + 2) +
positional_indexing_varint(len(value)) + value
+
+
+def positional_indexing_schema(name="id", nested=False):
+ # types.proto: DataType.long=7, .struct=21; Struct.fields=1;
+ # StructField: name=1, data_type=2, nullable=3, metadata=4.
+ data_type = positional_indexing_message(21, positional_indexing_message(1,
positional_indexing_field("inner", positional_indexing_message(7, b"")))) if
nested else positional_indexing_message(7, b"")
+ return positional_indexing_message(21, positional_indexing_message(1,
positional_indexing_field(name, data_type)))
+
+
+def positional_indexing_field(name, data_type):
+ return (positional_indexing_message(1, name) +
positional_indexing_message(2, data_type) + b"\x18\x01"
+ + positional_indexing_message(4, '{"note":"original"}'))
+
+
+def positional_indexing_serve(connection):
+ import grpc as positional_indexing_grpc
+ # A separate process also exercises columns/dtypes, whose first access may
+ # hold the client's GIL while waiting for the RPC.
+ class Service:
+ calls = 0
+ data_type = positional_indexing_schema()
+ failure = None
+
+ def analyze(self, request, context):
+ self.calls += 1
+ # Both clients serialize the UUID session identifier as field 1.
+ # Echo it so reference Python can verify response integrity.
+ assert request[0] == 10 and request[1] < 128
+ session = request[:2 + request[1]]
+ if self.failure == "rpc":
+
context.abort(positional_indexing_grpc.StatusCode.INVALID_ARGUMENT, "schema
unavailable")
+ if self.failure == "no-result":
+ return b""
+ if self.failure == "missing-schema":
+ return positional_indexing_message(2, b"")
+ if self.failure == "invalid-type":
+ return positional_indexing_message(2,
positional_indexing_message(1, b""))
+ # base.proto: AnalyzePlanResponse.schema=2, Schema.schema=1.
+ return session + positional_indexing_message(2,
positional_indexing_message(1, self.data_type))
+
+ service = Service()
+ with positional_indexing_ThreadPoolExecutor(max_workers=4) as executor:
+ server = positional_indexing_grpc.server(executor)
+ handler =
positional_indexing_grpc.unary_unary_rpc_method_handler(service.analyze)
+ release = positional_indexing_grpc.unary_unary_rpc_method_handler(
+ lambda request, context: request[:2 + request[1]])
+ config = positional_indexing_grpc.unary_unary_rpc_method_handler(
+ lambda request, context: request[:2 + request[1]] +
positional_indexing_message(
+ 2, positional_indexing_message(1,
"spark.python.sql.dataFrameDebugging.enabled")
+ + positional_indexing_message(2, "false")))
+
server.add_generic_rpc_handlers((positional_indexing_grpc.method_handlers_generic_handler(
+ "spark.connect.SparkConnectService", {
+ "AnalyzePlan": handler, "ReleaseSession": release, "Config":
config}),))
+ port = server.add_insecure_port("127.0.0.1:0")
+ server.start()
+ connection.send(port)
+ try:
+ while True:
+ command = connection.recv()
+ if command[0] == "stop":
+ break
+ if command[0] == "get":
+ connection.send(getattr(service, command[1]))
+ elif command[0] == "set":
+ setattr(service, command[1], command[2])
+ connection.send(None)
+ finally:
+ server.stop(0).wait()
+
+
+class positional_indexing_RemoteService:
+ def __init__(self, connection):
+ object.__setattr__(self, "connection", connection)
+
+ def __getattr__(self, name):
+ self.connection.send(("get", name))
+ assert self.connection.poll(10), "Mock server did not respond"
+ return self.connection.recv()
+
+ def __setattr__(self, name, value):
+ self.connection.send(("set", name, value))
+ assert self.connection.poll(10), "Mock server did not respond"
+ self.connection.recv()
+
+
[email protected]
+def positional_indexing_analyzed():
+ pytest.importorskip("grpc")
+ context = positional_indexing_multiprocessing.get_context("spawn")
+ parent, child = context.Pipe()
+ process = context.Process(target=positional_indexing_serve, args=(child,))
+ process.start()
+ spark = None
+ try:
+ assert parent.poll(20), "Mock server did not start"
+ port = parent.recv()
+ spark =
positional_indexing_SparkSession.builder.remote(f"sc://127.0.0.1:{port}").create()
+ yield spark, positional_indexing_RemoteService(parent)
+ finally:
+ if spark is not None:
+ spark.stop()
+ if process.is_alive():
+ parent.send(("stop",))
+ process.join(10)
+ if process.is_alive():
+ process.terminate()
+ process.join(10)
+ parent.close()
+ child.close()
+
+
+def positional_indexing_columns(*names):
+ return positional_indexing_message(21,
b"".join(positional_indexing_message(1, positional_indexing_field(name,
positional_indexing_message(7, b""))) for name in names))
+
+
[email protected]("index, name", [
+ (0, "first"), (1, "second"), (2, "last"),
+ (-1, "last"), (-2, "second"), (-3, "first"),
+ (False, "first"), (True, "second"),
+])
+def test_positional_indexing_positional_column(positional_indexing_analyzed,
index, name):
+ spark, service = positional_indexing_analyzed
+ service.data_type = positional_indexing_columns("first", "second", "last")
+ df = spark.range(10)
+ column = df[index]
+ assert isinstance(column, positional_indexing_Column)
+ assert repr(column) == repr(df[name])
+ assert service.calls == 1
+
+
[email protected]("names, index", [
+ (("a", "b"), 2), (("a", "b"), -3),
+ (("a",), 10**100), (("a",), -(10**100)),
+ ((), 0), ((), -1), ((), False), (("a",), True),
+])
+def test_positional_indexing_positional_bounds(positional_indexing_analyzed,
names, index):
+ spark, service = positional_indexing_analyzed
+ service.data_type = positional_indexing_columns(*names)
+ with pytest.raises(IndexError):
+ spark.range(10)[index]
+ assert service.calls == 1
+
+
+def
test_positional_indexing_integer_subclass_and_string_lookup(positional_indexing_analyzed):
+ class Position(int):
+ pass
+ class Name(str):
+ pass
+ spark, service = positional_indexing_analyzed
+ service.data_type = positional_indexing_columns("a.b", "with space", "a.b")
+ df = spark.range(10)
+ assert repr(df[Position(-2)]) == repr(df[Name("with space")])
+ assert repr(df[0]) == repr(df["a.b"])
+ before = service.calls
+ for name in ["missing", "*", "`a.b`", ""]:
+ assert isinstance(df[name], positional_indexing_Column)
+ assert service.calls == before
+
+
+def
test_positional_indexing_invalid_index_types_do_not_analyze(positional_indexing_analyzed):
+ from pyspark.errors import PySparkTypeError
+
+ class Position:
+ def __index__(self):
+ return 0
+ spark, service = positional_indexing_analyzed
+ df = spark.range(10)
+ for value in [0.0, None, b"id", slice(0, 1), Position()]:
+ # As PySpark, which takes a Column, an int, a list, a str or a tuple.
+ with pytest.raises(PySparkTypeError) as error:
+ df[value]
+ assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+ assert error.value.getMessageParameters() == {
+ "expected_type": "Column, int, list, str or tuple", "arg_name":
"item",
+ "arg_type": type(value).__name__}
+ assert service.calls == 0
+
+
+def
test_positional_indexing_columns_filter_and_sequences_select(positional_indexing_analyzed):
+ from pyspark.sql import DataFrame
+
+ # As PySpark: a Column filters the rows, and a list or tuple selects its
columns,
+ # without looking the schema up. The live tests check the rows.
+ spark, service = positional_indexing_analyzed
+ df = spark.range(10)
+ for item in [df["id"] > 1, ["id"], ("id", "id"), [["id"]], []]:
+ assert type(df[item]) is DataFrame
+ assert service.calls == 0
+
+
+def
test_positional_indexing_positional_propagates_schema_failure(positional_indexing_analyzed):
+ spark, service = positional_indexing_analyzed
+ service.failure = "rpc"
+ df = spark.range(10)
+ with pytest.raises(Exception, match="schema unavailable"):
+ df[0]
+ assert service.calls == 1
+ assert isinstance(df["id"], positional_indexing_Column)
+ assert service.calls == 1
+
+
# ------------------------------------------------- reader/writer in-place
setters
@pytest.fixture
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]