This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new b7ea970  [SPARK-60682][CONNECT] Include schema in ordinary DataFrame 
repr
b7ea970 is described below

commit b7ea9703ec5905714ef6a9549f0c9fb4f8d20604
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 16:33:38 2026 +0900

    [SPARK-60682][CONNECT] Include schema in ordinary DataFrame repr
    
    ### What changes were proposed in this pull request?
    
    `DataFrame.__repr__` now follows PySpark Connect's:
    
    - By default it is `DataFrame[name: type, ...]`, the names and 
`simpleString` types of the schema, through the DataFrame's cached schema. 
Before, it was the bare string `DataFrame`.
    - When the session's `spark.sql.repl.eagerEval.enabled` is `true`, it is 
the table of `_show_string`, with `spark.sql.repl.eagerEval.maxNumRows` rows 
and `spark.sql.repl.eagerEval.truncate`, as in PySpark.
    
    Like PySpark, it reads the three settings, in one request, on every call.
    
    ```python
    >>> spark.createDataFrame([(1, "a", [2])], "id long, s string, a 
array<int>")
    DataFrame[id: bigint, s: string, a: array<int>]
    >>> spark.conf.set("spark.sql.repl.eagerEval.enabled", "true")
    >>> spark.conf.set("spark.sql.repl.eagerEval.maxNumRows", "2")
    >>> spark.range(3)
    +---+
    | id|
    +---+
    |  0|
    |  1|
    +---+
    only showing top 2 rows
    ```
    
    ### Why are the changes needed?
    
    [SPARK-60682](https://issues.apache.org/jira/browse/SPARK-60682): the repr 
of every DataFrame was `DataFrame`, which said nothing about it, and eager 
evaluation was ignored.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `repr(df)` shows the schema, or the rows with eager evaluation 
enabled, as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests against a mock gRPC server check the repr of empty, 
flat, duplicate, Unicode, nested, collection, decimal, binary, date, timestamp 
and collated string schemas, the errors of a failed schema lookup and a 
successful retry, that the schema form reads the three settings once and 
executes nothing, and that eager evaluation executes the show-string query.
    - A new live test checks the tables with eager evaluation, with 
`maxNumRows` and `truncate`, and the schema form after the settings are unset. 
It also passes with official PySpark 4.3.0.
    - Against the current master, merged into this branch, and a local Spark 
4.3.0 server: the lib and every integration test of `apache-spark-connect` and 
`apache-spark-connect-core`, e2e included (771 passed), the build of all tests 
with `wasm-udf`, the offline Python suites (733 passed, 9 skipped), the live 
drop-in and Arrow suites (77 passed) and the live cases (10 passed). `cargo fmt 
--all --check` passes.
    
    Not in this change: `DataFrame._repr_html_`, PySpark's HTML table for 
notebooks.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6); the merge with master and the eager 
evaluation by Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #123 from dbtsai/codex/spark-60682-dataframe-repr.
    
    Lead-authored-by: DB Tsai <[email protected]>
    Co-authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  |  30 ++++-
 python/tests/test_dropin_e2e.py     |  18 +++
 python/tests/test_dropin_offline.py | 226 ++++++++++++++++++++++++++++++++++++
 3 files changed, 272 insertions(+), 2 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 7fbc38a..d41989d 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1588,8 +1588,34 @@ impl PyDataFrame {
         PyColumn::new(spark_connect::functions::col(item))
     }
 
-    fn __repr__(&self) -> String {
-        "DataFrame".to_string()
+    /// As PySpark's `DataFrame.__repr__`: the table of `_show_string` when 
the session's
+    /// `spark.sql.repl.eagerEval.enabled` is `true`, with its `maxNumRows` 
rows and its
+    /// `truncate`, and `DataFrame[name: type, ...]` otherwise. Like PySpark, 
it reads the
+    /// three settings, in one request, on every call.
+    fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
+        const EAGER_EVAL: [&str; 3] = [
+            "spark.sql.repl.eagerEval.enabled",
+            "spark.sql.repl.eagerEval.maxNumRows",
+            "spark.sql.repl.eagerEval.truncate",
+        ];
+        let conf = self.dataframe.spark_session().conf();
+        let values = py.detach(|| conf.get_many(&EAGER_EVAL)).to_pyerr()?;
+        if values[0].as_deref() == Some("true") {
+            // As PySpark's `int(...)` of each setting.
+            let int = py.import("builtins")?.getattr("int")?;
+            let n: i32 = int.call1((values[1].as_deref(),))?.extract()?;
+            let truncate: i32 = int.call1((values[2].as_deref(),))?.extract()?;
+            return py
+                .detach(|| self.dataframe.show_string(n, truncate, false))
+                .to_pyerr();
+        }
+        let fields = self
+            .dtypes(py)?
+            .into_iter()
+            .map(|(name, dtype)| format!("{name}: {dtype}"))
+            .collect::<Vec<_>>()
+            .join(", ");
+        Ok(format!("DataFrame[{fields}]"))
     }
 
     /// `DataFrame.mapInPandas` — map an iterator of pandas DataFrames.
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e1e380a..e468224 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -62,6 +62,24 @@ def test_conf_get_with_a_none_default(session):
         session.conf.get(key)
 
 
+def test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
+    df = session.range(3)
+    assert repr(df) == "DataFrame[id: bigint]"
+    keys = ["spark.sql.repl.eagerEval.enabled", 
"spark.sql.repl.eagerEval.maxNumRows",
+            "spark.sql.repl.eagerEval.truncate"]
+    try:
+        session.conf.set(keys[0], "true")
+        session.conf.set(keys[1], "2")
+        assert repr(df) == "+---+\n| id|\n+---+\n|  0|\n|  1|\n+---+\nonly 
showing top 2 rows"
+        session.conf.set(keys[2], "3")
+        assert repr(session.createDataFrame([("abcdef",)], "s string")) == (
+            "+---+\n|  s|\n+---+\n|abc|\n+---+\n")
+    finally:
+        for key in keys:
+            session.conf.unset(key)
+    assert repr(df) == "DataFrame[id: bigint]"
+
+
 def test_instr_and_regexp_replace_apply_their_optional_arguments(session):
     # Column b holds "c", so the literal "b" and the column b find different 
positions.
     df = session.createDataFrame([("abcabc", "c", "a", "z")], ["s", "b", "p", 
"r"])
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 70eb45d..e719248 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5274,6 +5274,232 @@ def test_create_auto_cdc_flow_scd2_track_history():
     assert flow.spark_conf == {"k": "v"}
 
 
+# ---------------------------------------------------------------- DataFrame 
repr
+
+def _repr_varint(value):
+    result = bytearray()
+    while value > 127:
+        result.append((value & 127) | 128)
+        value >>= 7
+    result.append(value)
+    return bytes(result)
+
+
+def _repr_message(field, value):
+    if isinstance(value, str):
+        value = value.encode()
+    return _repr_varint(field * 8 + 2) + _repr_varint(len(value)) + value
+
+
+def _repr_field(name, data_type):
+    return (_repr_message(1, name) + _repr_message(2, data_type) + b"\x18\x01"
+            + _repr_message(4, '{"note":"original"}'))
+
+
+def _repr_serve(connection):
+    import grpc
+    from concurrent.futures import ThreadPoolExecutor
+
+    # A separate process can serve schema RPCs while repr holds the client GIL.
+    class Service:
+        calls = 0
+        data_type = _repr_struct(("id", _repr_message(7, b"")))
+        failure = None
+        execute_calls = 0
+        configs = {"spark.sql.repl.eagerEval.enabled": "false",
+                   "spark.sql.repl.eagerEval.maxNumRows": "20",
+                   "spark.sql.repl.eagerEval.truncate": "20"}
+        config_keys = []
+
+        def execute(self, request, context):
+            self.execute_calls += 1
+            context.abort(grpc.StatusCode.UNIMPLEMENTED, "repr must not 
execute rows")
+
+        def config(self, request, context):
+            # ConfigRequest.session_id=1, operation=3, Operation.get=2, 
Get.keys=1;
+            # ConfigResponse.session_id=1, pairs=2, KeyValue.key=1, value=2.
+            request = parse_message(request)
+            keys = [key.decode() for key in
+                    parse_message(parse_message(request[3][0])[2][0]).get(1, 
[])]
+            self.config_keys = self.config_keys + [keys]
+            return _repr_message(1, request[1][0]) + b"".join(
+                _repr_message(2, _repr_message(1, key) + _repr_message(2, 
self.configs[key]))
+                for key in keys)
+
+        def analyze(self, request, context):
+            self.calls += 1
+            if self.failure == "rpc":
+                context.abort(grpc.StatusCode.INVALID_ARGUMENT, "schema 
unavailable")
+            if self.failure == "no-result":
+                return b""
+            if self.failure == "missing-schema":
+                return _repr_message(2, b"")
+            if self.failure == "invalid-type":
+                return _repr_message(2, _repr_message(1, b""))
+            # base.proto: AnalyzePlanResponse.schema=2, Schema.schema=1.
+            return _repr_message(2, _repr_message(1, self.data_type))
+
+    service = Service()
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        server = grpc.server(executor)
+        handler = grpc.unary_unary_rpc_method_handler(service.analyze)
+        server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
+            "spark.connect.SparkConnectService", {"AnalyzePlan": handler,
+                "ExecutePlan": 
grpc.unary_stream_rpc_method_handler(service.execute),
+                "Config": 
grpc.unary_unary_rpc_method_handler(service.config)}),))
+        port = server.add_insecure_port("127.0.0.1:0")
+        server.start()
+        connection.send(port)
+        try:
+            while True:
+                command = connection.recv()
+                if command[0] == "stop":
+                    break
+                if command[0] == "get":
+                    connection.send(getattr(service, command[1]))
+                elif command[0] == "set":
+                    setattr(service, command[1], command[2])
+                    connection.send(None)
+        finally:
+            server.stop(0).wait()
+
+
+class _ReprRemoteService:
+    def __init__(self, connection):
+        object.__setattr__(self, "connection", connection)
+
+    def __getattr__(self, name):
+        self.connection.send(("get", name))
+        assert self.connection.poll(10), "Mock server did not respond"
+        return self.connection.recv()
+
+    def __setattr__(self, name, value):
+        self.connection.send(("set", name, value))
+        assert self.connection.poll(10), "Mock server did not respond"
+        self.connection.recv()
+
+
[email protected]
+def repr_analyzed():
+    import multiprocessing
+    from pyspark.sql import SparkSession
+
+    pytest.importorskip("grpc")
+    context = multiprocessing.get_context("spawn")
+    parent, child = context.Pipe()
+    process = context.Process(target=_repr_serve, args=(child,))
+    process.start()
+    try:
+        assert parent.poll(20), "Mock server did not start"
+        port = parent.recv()
+        spark = 
SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate()
+        yield spark, _ReprRemoteService(parent)
+    finally:
+        if process.is_alive():
+            parent.send(("stop",))
+        process.join(10)
+        if process.is_alive():
+            process.terminate()
+            process.join(10)
+        parent.close()
+        child.close()
+
+
+def _repr_struct(*fields):
+    return _repr_message(21, b"".join(_repr_message(1, _repr_field(name, 
dtype)) for name, dtype in fields))
+
+
[email protected]("schema, expected", [
+    (_repr_struct(), "DataFrame[]"),
+    (_repr_struct(("id", _repr_message(7, b""))), "DataFrame[id: bigint]"),
+    (_repr_struct(("a.b", _repr_message(6, b"")), ("with space", 
_repr_message(11, b"")),
+             ("a.b", _repr_message(9, b"")), ("雪`[]", _repr_message(3, b""))),
+     "DataFrame[a.b: int, with space: string, a.b: double, 雪`[]: boolean]"),
+    (_repr_struct(("nested", _repr_struct(("inner", _repr_message(7, b"")),
+                               ("text", _repr_message(11, b""))))),
+     "DataFrame[nested: struct<inner:bigint,text:string>]"),
+    (_repr_struct(("items", _repr_message(20, _repr_message(1, 
_repr_message(6, b"")) + b"\x10\x01")),
+             ("lookup", _repr_message(22, _repr_message(1, _repr_message(11, 
b""))
+                                 + _repr_message(2, _repr_message(7, b"")) + 
b"\x18\x01")),
+             ("amount", _repr_message(10, b"\x08\x02\x10\x0a"))),
+     "DataFrame[items: array<int>, lookup: map<string,bigint>, amount: 
decimal(10,2)]"),
+    (_repr_struct(("missing", _repr_message(1, b"")), ("raw", _repr_message(2, 
b"")),
+             ("day", _repr_message(14, b"")), ("ts", _repr_message(15, b"")),
+             ("local", _repr_message(16, b"")),
+             ("label", _repr_message(11, _repr_message(2, "UNICODE")))),
+     "DataFrame[missing: void, raw: binary, day: date, ts: timestamp, "
+     "local: timestamp_ntz, label: string collate UNICODE]"),
+])
+def test_repr_formats_schema_without_executing_rows(repr_analyzed, schema, 
expected):
+    spark, service = repr_analyzed
+    service.data_type = schema
+    df = spark.range(10)
+    assert service.calls == 0
+    assert repr(df) == expected
+    assert service.calls == 1
+    assert service.execute_calls == 0
+    assert service.config_keys == [_REPR_EAGER_EVAL_KEYS]
+
+
[email protected]("failure, message", [
+    ("rpc", "schema unavailable"),
+    ("no-result", "no schema in response"),
+    ("missing-schema", "Schema is missing"),
+    ("invalid-type", "DataType kind not set"),
+])
+def test_repr_propagates_schema_errors_and_can_retry(repr_analyzed, failure, 
message):
+    spark, service = repr_analyzed
+    df = spark.range(10)
+    service.failure = failure
+    with pytest.raises(Exception, match=message):
+        repr(df)
+    assert service.calls == 1
+    service.failure = None
+    assert repr(df) == "DataFrame[id: bigint]"
+    assert service.calls == 2
+    assert service.execute_calls == 0
+
+
+_REPR_EAGER_EVAL_KEYS = ["spark.sql.repl.eagerEval.enabled",
+                         "spark.sql.repl.eagerEval.maxNumRows",
+                         "spark.sql.repl.eagerEval.truncate"]
+
+
+def test_repr_executes_the_show_string_with_eager_evaluation(repr_analyzed):
+    # As PySpark: with eager evaluation, repr is the table of _show_string, so 
it executes
+    # the DataFrame (which this mock refuses), rather than formatting the 
schema.
+    spark, service = repr_analyzed
+    service.configs = {"spark.sql.repl.eagerEval.enabled": "true",
+                       "spark.sql.repl.eagerEval.maxNumRows": "2",
+                       "spark.sql.repl.eagerEval.truncate": "3"}
+    with pytest.raises(Exception, match="repr must not execute rows"):
+        repr(spark.range(10))
+    assert service.execute_calls == 1
+    assert service.calls == 0
+    assert service.config_keys == [_REPR_EAGER_EVAL_KEYS]
+
+
+def test_repr_rejects_non_struct_schema(repr_analyzed):
+    spark, service = repr_analyzed
+    service.data_type = _repr_message(7, b"")
+    with pytest.raises(Exception, match="Schema is not a struct type"):
+        repr(spark.range(10))
+    assert service.execute_calls == 0
+
+
+def test_repr_handles_wide_schema_and_independent_dataframe(repr_analyzed):
+    spark, service = repr_analyzed
+    service.data_type = _repr_struct(*[(f"c{i}", _repr_message(7, b"")) for i 
in range(256)])
+    df = spark.range(10)
+    assert repr(df) == "DataFrame[" + ", ".join(f"c{i}: bigint" for i in 
range(256)) + "]"
+    # A new independent DataFrame resolves updated server metadata even when
+    # another change caches the original DataFrame's schema.
+    service.data_type = _repr_struct(("renamed", _repr_message(11, b"")))
+    assert repr(spark.range(10)) == "DataFrame[renamed: string]"
+    assert service.calls == 2
+    assert service.execute_calls == 0
+
+
 # --------------------------------------------------------------------------- 
ExecutePlan (mock)
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to