This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 0ac7de1 [SPARK-60681][CONNECT] Add DataFrame inspection completions
0ac7de1 is described below
commit 0ac7de17188b883fffcf7278f1135ff5a3aa9890
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:32:41 2026 +0900
[SPARK-60681][CONNECT] Add DataFrame inspection completions
### What changes were proposed in this pull request?
Add `DataFrame.__dir__` and `DataFrame._ipython_key_completions_`, as in
PySpark Connect:
- `dir(df)` returns the sorted, deduplicated attributes of the `DataFrame`
class plus all column names (PySpark Connect's `sorted(set(dir(DataFrame)) |
set(self.columns))`).
- `df._ipython_key_completions_()` returns the column names in schema
order, duplicates and non-identifiers included.
Both look the schema up through the DataFrame's cached schema, so a
repeated call sends no RPC, and both raise the error of a failed schema lookup,
as PySpark does.
### Why are the changes needed?
[SPARK-60681](https://issues.apache.org/jira/browse/SPARK-60681): for a
DataFrame with a column `c0`, `dir(df)` omitted `c0` and
`_ipython_key_completions_()` raised an `AttributeError`, so IPython and
Jupyter could not complete column names after `df.` or in `df["`.
### Does this PR introduce _any_ user-facing change?
Yes. `dir(df)` lists the columns, and IPython completes column names in
`df["...`, as in PySpark.
### How was this patch tested?
- New offline tests against a mock gRPC server in a separate process check
ordinary, non-identifier, Unicode, empty and duplicate names, names that
collide with methods, a new list per call, schema errors and a successful
retry, and the RPCs of the first and repeated calls.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (771 passed), the build of all tests
with `wasm-udf`, the offline Python suites (733 passed, 9 skipped), the live
drop-in and Arrow suites (76 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #124 from dbtsai/codex/spark-60681-dataframe-completions.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 15 +++
python/tests/test_dropin_offline.py | 177 ++++++++++++++++++++++++++++++++++++
2 files changed, 192 insertions(+)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 944a41f..7fbc38a 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1206,6 +1206,21 @@ impl PyDataFrame {
Ok(PyColumn::new(spark_connect::functions::col(name)))
}
+ /// Class attributes and all column names, matching Python Connect.
+ fn __dir__(&self, py: Python<'_>) -> PyResult<Vec<String>> {
+ // Inspect the class rather than the instance to avoid calling __dir__
again.
+ let mut attrs = py.get_type::<Self>().dir()?.extract::<Vec<String>>()?;
+ attrs.extend(self.columns(py)?);
+ attrs.sort();
+ attrs.dedup();
+ Ok(attrs)
+ }
+
+ /// Bracket completion keeps column order, duplicates and non-identifiers.
+ fn _ipython_key_completions_(&self, py: Python<'_>) ->
PyResult<Vec<String>> {
+ self.columns(py)
+ }
+
/// Aggregate over the whole DataFrame (shorthand for
`groupBy().agg(...)`).
///
/// Mirrors `pyspark.sql.DataFrame.agg(*exprs)`.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b3733b3..f42f87c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4618,6 +4618,183 @@ def test_arrow_input_large_upload_live_server():
spark.stop()
+# ------------------------------------------------------- DataFrame completion
+
+def _completion_varint(value):
+ result = bytearray()
+ while value > 127:
+ result.append((value & 127) | 128)
+ value >>= 7
+ result.append(value)
+ return bytes(result)
+
+
+def _completion_message(field, value):
+ if isinstance(value, str):
+ value = value.encode()
+ return _completion_varint(field * 8 + 2) + _completion_varint(len(value))
+ value
+
+
+def _completion_schema(names=("id",)):
+ # DataType.struct=21, Struct.fields=1, DataType.long=7.
+ return _completion_message(21, b"".join(_completion_message(1,
_completion_field(name, _completion_message(7, b"")))
+ for name in names))
+
+
+def _completion_field(name, data_type):
+ return (_completion_message(1, name) + _completion_message(2, data_type) +
b"\x18\x01"
+ + _completion_message(4, '{"note":"original"}'))
+
+
+def _completion_serve(connection):
+ from concurrent.futures import ThreadPoolExecutor
+ import grpc
+
+ # A separate process also exercises columns/dtypes, whose first access may
+ # hold the client's GIL while waiting for the RPC.
+ class Service:
+ calls = 0
+ data_type = _completion_schema()
+ failure = None
+
+ def analyze(self, request, context):
+ self.calls += 1
+ if self.failure == "rpc":
+ context.abort(grpc.StatusCode.INVALID_ARGUMENT, "schema
unavailable")
+ if self.failure == "no-result":
+ return b""
+ if self.failure == "missing-schema":
+ return _completion_message(2, b"")
+ if self.failure == "invalid-type":
+ return _completion_message(2, _completion_message(1, b""))
+ # base.proto: AnalyzePlanResponse.schema=2, Schema.schema=1.
+ return _completion_message(2, _completion_message(1,
self.data_type))
+
+ service = Service()
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ server = grpc.server(executor)
+ handler = grpc.unary_unary_rpc_method_handler(service.analyze)
+ server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
+ "spark.connect.SparkConnectService", {"AnalyzePlan": handler}),))
+ port = server.add_insecure_port("127.0.0.1:0")
+ server.start()
+ connection.send(port)
+ try:
+ while True:
+ command = connection.recv()
+ if command[0] == "stop":
+ break
+ if command[0] == "get":
+ connection.send(getattr(service, command[1]))
+ elif command[0] == "set":
+ setattr(service, command[1], command[2])
+ connection.send(None)
+ finally:
+ server.stop(0).wait()
+
+
+class _CompletionRemoteService:
+ def __init__(self, connection):
+ object.__setattr__(self, "connection", connection)
+
+ def __getattr__(self, name):
+ self.connection.send(("get", name))
+ assert self.connection.poll(10), "Mock server did not respond"
+ return self.connection.recv()
+
+ def __setattr__(self, name, value):
+ self.connection.send(("set", name, value))
+ assert self.connection.poll(10), "Mock server did not respond"
+ self.connection.recv()
+
+
[email protected]
+def completion_analyzed():
+ import multiprocessing
+ from pyspark.sql import SparkSession
+
+ pytest.importorskip("grpc")
+ context = multiprocessing.get_context("spawn")
+ parent, child = context.Pipe()
+ process = context.Process(target=_completion_serve, args=(child,))
+ process.start()
+ try:
+ assert parent.poll(20), "Mock server did not start"
+ port = parent.recv()
+ spark =
SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate()
+ yield spark, _CompletionRemoteService(parent)
+ finally:
+ if process.is_alive():
+ parent.send(("stop",))
+ process.join(10)
+ if process.is_alive():
+ process.terminate()
+ process.join(10)
+ parent.close()
+ child.close()
+
+
[email protected]("names", [
+ ["z", "a"],
+ ["with space", "a.b", "with`tick", "", "123", "class", "雪", "_private"],
+ ["id", "id", "select", "columns", "__dir__", "_ipython_key_completions_"],
+ [],
+])
+def test_completion_matches_python_connect(completion_analyzed, names):
+ from pyspark.sql import Column
+
+ spark, service = completion_analyzed
+ service.data_type = _completion_schema(names)
+ df = spark.range(10)
+ assert df._ipython_key_completions_() == names
+ assert service.calls == 1
+ expected = sorted(set(dir(type(df))) | set(names))
+ # Each first access must analyze its own fresh plan, with or without
caching.
+ assert spark.range(10).__dir__() == expected
+ assert service.calls == 2
+ assert dir(spark.range(10)) == expected
+ assert service.calls == 3
+ assert callable(df.select)
+ assert callable(df.__dir__)
+ assert callable(df._ipython_key_completions_)
+ assert df.columns == names
+ for name in names:
+ assert isinstance(df[name], Column)
+ assert df.schema.fieldNames() == names
+
+
+def test_completion_lists_are_independent(completion_analyzed):
+ spark, service = completion_analyzed
+ df = spark.range(10)
+ df._ipython_key_completions_().append("extra")
+ df.__dir__().clear()
+ assert df._ipython_key_completions_() == ["id"]
+ assert "select" in dir(df)
+ service.data_type = _completion_schema(["changed"])
+ assert spark.range(10)._ipython_key_completions_() == ["changed"]
+
+
[email protected]("method", ["__dir__", "_ipython_key_completions_"])
[email protected]("failure", ["rpc", "no-result", "missing-schema",
"invalid-type"])
+def
test_completion_propagates_schema_errors_and_can_retry(completion_analyzed,
method, failure):
+ spark, service = completion_analyzed
+ df = spark.range(10)
+ service.failure = failure
+ message = {
+ "rpc": "schema unavailable",
+ "no-result": "no schema in response",
+ "missing-schema": "Schema is missing",
+ "invalid-type": "DataType kind not set",
+ }[failure]
+ for count in (1, 2):
+ with pytest.raises(Exception, match=message):
+ getattr(df, method)()
+ assert service.calls == count
+ service.failure = None
+ assert "id" in getattr(df, method)()
+ assert service.calls == 3
+
+
# ---------------------------------------------------------------------------
4.3.0 gaps
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]