This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 1eaffb5  [SPARK-61110][CONNECT][PYTHON] Add DataFrame._repr_html_, the 
HTML table of eager evaluation in notebooks, in the Python client as PySpark 
does
1eaffb5 is described below

commit 1eaffb5afa0d7d509ed0588bd345c4d30d36e3fe
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 19:27:33 2026 +0900

    [SPARK-61110][CONNECT][PYTHON] Add DataFrame._repr_html_, the HTML table of 
eager evaluation in notebooks, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61110](https://issues.apache.org/jira/browse/SPARK-61110): 
`DataFrame._repr_html_` is added, as PySpark Connect's.
    
    Jupyter calls `_repr_html_` to display a DataFrame. As in PySpark Connect:
    - With `spark.sql.repl.eagerEval.enabled`, it returns the HTML table of the 
first `maxNumRows` rows, with values truncated to `truncate` characters. The 
server formats the table, of an `HtmlString` relation.
    - Otherwise it returns None.
    - Once it has been called, `__repr__` of the DataFrame is its 
`DataFrame[...]` string (PySpark's `_support_repr_html`). So a notebook does 
not show the table twice.
    
    The core gains the `HtmlString` plan and `DataFrame::html_string`, as it 
has `ShowString` and `show_string`.
    
    ### Why are the changes needed?
    
    The client lacked `_repr_html_`. Attribute access took it as a column, so 
`df._repr_html_()` raised `TypeError: 'Column' object is not callable`. A 
notebook with eager evaluation showed the text table of `__repr__` instead of 
the HTML table.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Notebooks show DataFrames as with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks `_repr_html_` without and with eager evaluation, 
and `__repr__` after it. It passes with official PySpark 4.3.0 and fails 
without this change.
    - A new offline test checks, against a mock Connect server:
      - the `HtmlString` relation sent, of `maxNumRows` and `truncate`
      - the table returned
      - `__repr__` before and after `_repr_html_`
    - On a local server, 41 cases give the same results as official PySpark 
4.3.0. They cover `show`, `printSchema`, `explain`, `__repr__`, `_repr_html_` 
and `describe`/`summary`, with eager evaluation, truncation and maximum rows. 
The only difference is the expression ids in `explain`.
    - The lib tests of both crates (589 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2056 passed, 12 skipped), and the 
live drop-in and Arrow suites (160 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #306 from HyukjinKwon/SPARK-61110-repr-html.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    | 70 +++++++++++++++++++++++++++--------
 crates/spark-connect/src/dataframe.rs | 19 ++++++++++
 crates/spark-connect/src/plan.rs      | 20 ++++++++++
 python/tests/test_dropin_e2e.py       | 14 +++++++
 python/tests/test_dropin_offline.py   | 32 ++++++++++++++++
 5 files changed, 139 insertions(+), 16 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 597c843..b0b96ee 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -46,8 +46,18 @@ fn flatten_str_cols(cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<Vec<String>> {
 #[pyclass(name = "DataFrame", module = "pyspark.sql.connect.dataframe", 
subclass)]
 pub struct PyDataFrame {
     pub(crate) dataframe: DataFrame,
+    /// Whether `_repr_html_` was called, as PySpark's `_support_repr_html`: 
then `__repr__` is
+    /// the string of the schema, so that a notebook does not show the 
DataFrame twice.
+    support_repr_html: std::sync::atomic::AtomicBool,
 }
 
+/// The settings of the eager evaluation of `__repr__` and `_repr_html_`.
+const EAGER_EVAL: [&str; 3] = [
+    "spark.sql.repl.eagerEval.enabled",
+    "spark.sql.repl.eagerEval.maxNumRows",
+    "spark.sql.repl.eagerEval.truncate",
+];
+
 impl PyDataFrame {
     /// `other`, a DataFrame of the session of this one, as PySpark's 
`_check_same_session`
     /// takes the DataFrame of a method: else `SESSION_NOT_SAME`, as for what 
is not a
@@ -180,7 +190,24 @@ impl PyDataFrame {
     }
 
     pub fn new(dataframe: DataFrame) -> Self {
-        PyDataFrame { dataframe }
+        PyDataFrame {
+            dataframe,
+            support_repr_html: std::sync::atomic::AtomicBool::new(false),
+        }
+    }
+
+    /// The number of rows and the truncation of the eager evaluation, as 
PySpark's `int` of
+    /// each setting, if it is enabled.
+    fn eager_eval(&self, py: Python<'_>) -> PyResult<Option<(i32, i32)>> {
+        let conf = self.dataframe.spark_session().conf();
+        let values = py.detach(|| conf.get_many(&EAGER_EVAL)).to_pyerr()?;
+        if values[0].as_deref() != Some("true") {
+            return Ok(None);
+        }
+        let int = py.import("builtins")?.getattr("int")?;
+        let n: i32 = int.call1((values[1].as_deref(),))?.extract()?;
+        let truncate: i32 = int.call1((values[2].as_deref(),))?.extract()?;
+        Ok(Some((n, truncate)))
     }
 
     /// The column at the 0-based index `index`, as PySpark's `df[index]`: 
`col` of the name of
@@ -2340,21 +2367,16 @@ impl PyDataFrame {
     /// `truncate`, and `DataFrame[name: type, ...]` otherwise. Like PySpark, 
it reads the
     /// three settings, in one request, on every call.
     fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
-        const EAGER_EVAL: [&str; 3] = [
-            "spark.sql.repl.eagerEval.enabled",
-            "spark.sql.repl.eagerEval.maxNumRows",
-            "spark.sql.repl.eagerEval.truncate",
-        ];
-        let conf = self.dataframe.spark_session().conf();
-        let values = py.detach(|| conf.get_many(&EAGER_EVAL)).to_pyerr()?;
-        if values[0].as_deref() == Some("true") {
-            // As PySpark's `int(...)` of each setting.
-            let int = py.import("builtins")?.getattr("int")?;
-            let n: i32 = int.call1((values[1].as_deref(),))?.extract()?;
-            let truncate: i32 = int.call1((values[2].as_deref(),))?.extract()?;
-            return py
-                .detach(|| self.dataframe.show_string(n, truncate, false))
-                .to_pyerr();
+        // As PySpark's: once `_repr_html_` was called, the string of the 
schema.
+        if !self
+            .support_repr_html
+            .load(std::sync::atomic::Ordering::Relaxed)
+        {
+            if let Some((n, truncate)) = self.eager_eval(py)? {
+                return py
+                    .detach(|| self.dataframe.show_string(n, truncate, false))
+                    .to_pyerr();
+            }
         }
         let fields = self
             .dtypes(py)?
@@ -2365,6 +2387,22 @@ impl PyDataFrame {
         Ok(format!("DataFrame[{fields}]"))
     }
 
+    /// The HTML table of the DataFrame, which notebooks show, as PySpark 
Connect's
+    /// `_repr_html_`: with `spark.sql.repl.eagerEval.enabled`, the table of 
the first
+    /// `maxNumRows` rows, values truncated to `truncate` characters, which 
the server formats
+    /// (`HtmlString`); else None. Once called, `__repr__` is the string of 
the schema.
+    fn _repr_html_(&self, py: Python<'_>) -> PyResult<Option<String>> {
+        self.support_repr_html
+            .store(true, std::sync::atomic::Ordering::Relaxed);
+        match self.eager_eval(py)? {
+            Some((n, truncate)) => py
+                .detach(|| self.dataframe.html_string(n, truncate))
+                .to_pyerr()
+                .map(Some),
+            None => Ok(None),
+        }
+    }
+
     /// `DataFrame.mapInPandas` — map an iterator of pandas DataFrames, in a 
stage of the
     /// resource profile `profile`, if any, as PySpark's.
     #[pyo3(name = "mapInPandas", signature = (func, schema, barrier = false, 
profile = None))]
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index fc826d5..708c3bc 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -962,6 +962,25 @@ impl DataFrame {
         }
     }
 
+    /// The HTML table of the first `n` rows, values truncated to `truncate` 
characters (0 for
+    /// none), as the server formats it. Mirrors the table of 
`DataFrame._repr_html_()`.
+    pub fn html_string(&self, n: i32, truncate: i32) -> Result<String> {
+        let html = DataFrame::new(
+            self.session.clone(),
+            LogicalPlan::HtmlString {
+                input: Arc::clone(&self.plan),
+                num_rows: n,
+                truncate,
+            },
+        );
+        match html.collect()?.first().map(|row| row.values()) {
+            Some([Value::String(table)]) => Ok(table.clone()),
+            _ => Err(SparkError::connect_msg(
+                "HtmlString returned no table of one string",
+            )),
+        }
+    }
+
     /// Print the first `n` rows as the server formats them, with values 
truncated to 20
     /// characters. Mirrors `DataFrame.show(n)`.
     pub fn show(&self, n: usize) -> Result<()> {
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index bce9c66..c11521d 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -106,6 +106,13 @@ pub enum LogicalPlan {
         truncate: i32,
         vertical: bool,
     },
+    /// HtmlString: the HTML table of `df._repr_html_()`, which the server 
formats as one
+    /// string: `num_rows` rows, values truncated to `truncate` characters (0 
for none).
+    HtmlString {
+        input: Arc<LogicalPlan>,
+        num_rows: i32,
+        truncate: i32,
+    },
     /// Deduplicate: `df.distinct()` or `df.dropDuplicates()`.
     Deduplicate {
         input: Arc<LogicalPlan>,
@@ -854,6 +861,19 @@ impl LogicalPlan {
                 relation.rel_type = 
Some(proto::relation::RelType::ShowString(Box::new(show)));
             }
 
+            LogicalPlan::HtmlString {
+                input,
+                num_rows,
+                truncate,
+            } => {
+                let html = proto::HtmlString {
+                    input: Some(Box::new(relation_of(input, plan_ids))),
+                    num_rows: *num_rows,
+                    truncate: *truncate,
+                };
+                relation.rel_type = 
Some(proto::relation::RelType::HtmlString(Box::new(html)));
+            }
+
             LogicalPlan::Deduplicate {
                 input,
                 all_columns_as_keys,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d7dc597..060e0cc 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1935,6 +1935,20 @@ def 
test_sql_fails_at_a_statement_that_does_not_parse(session):
         session.sql("SELEC 1")
 
 
+def test_repr_html_is_the_html_table_of_eager_evaluation_as_pyspark(session):
+    # As PySpark Connect's: _repr_html_ is None without eager evaluation, else 
the HTML table of
+    # the DataFrame, after which __repr__ is the string of the schema.
+    df = session.range(2)
+    assert df._repr_html_() is None
+    session.conf.set("spark.sql.repl.eagerEval.enabled", "true")
+    try:
+        html = df._repr_html_()
+        assert html.startswith("<table border='1'>") and "<th>id</th>" in html
+        assert repr(df) == "DataFrame[id: bigint]"
+        assert "+---+" in repr(session.range(2))
+    finally:
+        session.conf.unset("spark.sql.repl.eagerEval.enabled")
+
 def test_sql_formats_the_query_with_keyword_arguments_as_pyspark(session):
     from decimal import Decimal
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 1f546c0..70dad79 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9458,6 +9458,38 @@ def _repr_struct(*fields):
     return _repr_message(21, b"".join(_repr_message(1, _repr_field(name, 
dtype)) for name, dtype in fields))
 
 
+def 
test_repr_html_is_the_html_table_of_eager_evaluation_as_pyspark(mock_connect):
+    # As PySpark Connect's: _repr_html_ is None without eager evaluation, else 
the HTML table
+    # that the server formats of an HtmlString relation of maxNumRows rows and 
truncate; once
+    # it was called, __repr__ is the string of the schema.
+    import pyarrow as pa
+
+    spark, service = mock_connect
+    eager = {"spark.sql.repl.eagerEval.enabled": "false",
+             "spark.sql.repl.eagerEval.maxNumRows": "7",
+             "spark.sql.repl.eagerEval.truncate": "3"}
+    service.confs = eager
+    df = spark.range(1)
+    assert df._repr_html_() is None
+    service.confs = {**eager, "spark.sql.repl.eagerEval.enabled": "true"}
+    table = pa.table({"html_string": ["<table/>"]})
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    # ExecutePlanResponse.arrow_batch=2: ArrowBatch.row_count=1, data=2.
+    service.execute_payload = message_field(
+        2, varint_field(1, 1) + message_field(2, sink.getvalue().to_pybytes()))
+    assert df._repr_html_() == "<table/>"
+    # ExecutePlanRequest.plan=3, Plan.root=1, Relation.html_string=35: 
HtmlString.num_rows=2,
+    # truncate=3.
+    plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+    html = parse_message(parse_message(plan[1][0])[35][0])
+    assert html[2] == [7] and html[3] == [3]
+    assert repr(df) == "DataFrame[id: bigint]"
+    # A DataFrame of no _repr_html_ shows the table of eager evaluation.
+    assert repr(spark.range(1)) == "<table/>"
+
+
 @pytest.mark.parametrize("schema, expected", [
     (_repr_struct(), "DataFrame[]"),
     (_repr_struct(("id", _repr_message(7, b""))), "DataFrame[id: bigint]"),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to