This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 1eaffb5 [SPARK-61110][CONNECT][PYTHON] Add DataFrame._repr_html_, the
HTML table of eager evaluation in notebooks, in the Python client as PySpark
does
1eaffb5 is described below
commit 1eaffb5afa0d7d509ed0588bd345c4d30d36e3fe
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 19:27:33 2026 +0900
[SPARK-61110][CONNECT][PYTHON] Add DataFrame._repr_html_, the HTML table of
eager evaluation in notebooks, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61110](https://issues.apache.org/jira/browse/SPARK-61110):
`DataFrame._repr_html_` is added, as PySpark Connect's.
Jupyter calls `_repr_html_` to display a DataFrame. As in PySpark Connect:
- With `spark.sql.repl.eagerEval.enabled`, it returns the HTML table of the
first `maxNumRows` rows, with values truncated to `truncate` characters. The
server formats the table, of an `HtmlString` relation.
- Otherwise it returns None.
- Once it has been called, `__repr__` of the DataFrame is its
`DataFrame[...]` string (PySpark's `_support_repr_html`). So a notebook does
not show the table twice.
The core gains the `HtmlString` plan and `DataFrame::html_string`, as it
has `ShowString` and `show_string`.
### Why are the changes needed?
The client lacked `_repr_html_`. Attribute access took it as a column, so
`df._repr_html_()` raised `TypeError: 'Column' object is not callable`. A
notebook with eager evaluation showed the text table of `__repr__` instead of
the HTML table.
### Does this PR introduce _any_ user-facing change?
Yes. Notebooks show DataFrames as with PySpark 4.3.0.
### How was this patch tested?
- A new live test checks `_repr_html_` without and with eager evaluation,
and `__repr__` after it. It passes with official PySpark 4.3.0 and fails
without this change.
- A new offline test checks, against a mock Connect server:
- the `HtmlString` relation sent, of `maxNumRows` and `truncate`
- the table returned
- `__repr__` before and after `_repr_html_`
- On a local server, 41 cases give the same results as official PySpark
4.3.0. They cover `show`, `printSchema`, `explain`, `__repr__`, `_repr_html_`
and `describe`/`summary`, with eager evaluation, truncation and maximum rows.
The only difference is the expression ids in `explain`.
- The lib tests of both crates (589 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2056 passed, 12 skipped), and the
live drop-in and Arrow suites (160 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #306 from HyukjinKwon/SPARK-61110-repr-html.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 70 +++++++++++++++++++++++++++--------
crates/spark-connect/src/dataframe.rs | 19 ++++++++++
crates/spark-connect/src/plan.rs | 20 ++++++++++
python/tests/test_dropin_e2e.py | 14 +++++++
python/tests/test_dropin_offline.py | 32 ++++++++++++++++
5 files changed, 139 insertions(+), 16 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 597c843..b0b96ee 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -46,8 +46,18 @@ fn flatten_str_cols(cols: Vec<Bound<'_, PyAny>>) ->
PyResult<Vec<String>> {
#[pyclass(name = "DataFrame", module = "pyspark.sql.connect.dataframe",
subclass)]
pub struct PyDataFrame {
pub(crate) dataframe: DataFrame,
+ /// Whether `_repr_html_` was called, as PySpark's `_support_repr_html`:
then `__repr__` is
+ /// the string of the schema, so that a notebook does not show the
DataFrame twice.
+ support_repr_html: std::sync::atomic::AtomicBool,
}
+/// The settings of the eager evaluation of `__repr__` and `_repr_html_`.
+const EAGER_EVAL: [&str; 3] = [
+ "spark.sql.repl.eagerEval.enabled",
+ "spark.sql.repl.eagerEval.maxNumRows",
+ "spark.sql.repl.eagerEval.truncate",
+];
+
impl PyDataFrame {
/// `other`, a DataFrame of the session of this one, as PySpark's
`_check_same_session`
/// takes the DataFrame of a method: else `SESSION_NOT_SAME`, as for what
is not a
@@ -180,7 +190,24 @@ impl PyDataFrame {
}
pub fn new(dataframe: DataFrame) -> Self {
- PyDataFrame { dataframe }
+ PyDataFrame {
+ dataframe,
+ support_repr_html: std::sync::atomic::AtomicBool::new(false),
+ }
+ }
+
+ /// The number of rows and the truncation of the eager evaluation, as
PySpark's `int` of
+ /// each setting, if it is enabled.
+ fn eager_eval(&self, py: Python<'_>) -> PyResult<Option<(i32, i32)>> {
+ let conf = self.dataframe.spark_session().conf();
+ let values = py.detach(|| conf.get_many(&EAGER_EVAL)).to_pyerr()?;
+ if values[0].as_deref() != Some("true") {
+ return Ok(None);
+ }
+ let int = py.import("builtins")?.getattr("int")?;
+ let n: i32 = int.call1((values[1].as_deref(),))?.extract()?;
+ let truncate: i32 = int.call1((values[2].as_deref(),))?.extract()?;
+ Ok(Some((n, truncate)))
}
/// The column at the 0-based index `index`, as PySpark's `df[index]`:
`col` of the name of
@@ -2340,21 +2367,16 @@ impl PyDataFrame {
/// `truncate`, and `DataFrame[name: type, ...]` otherwise. Like PySpark,
it reads the
/// three settings, in one request, on every call.
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
- const EAGER_EVAL: [&str; 3] = [
- "spark.sql.repl.eagerEval.enabled",
- "spark.sql.repl.eagerEval.maxNumRows",
- "spark.sql.repl.eagerEval.truncate",
- ];
- let conf = self.dataframe.spark_session().conf();
- let values = py.detach(|| conf.get_many(&EAGER_EVAL)).to_pyerr()?;
- if values[0].as_deref() == Some("true") {
- // As PySpark's `int(...)` of each setting.
- let int = py.import("builtins")?.getattr("int")?;
- let n: i32 = int.call1((values[1].as_deref(),))?.extract()?;
- let truncate: i32 = int.call1((values[2].as_deref(),))?.extract()?;
- return py
- .detach(|| self.dataframe.show_string(n, truncate, false))
- .to_pyerr();
+ // As PySpark's: once `_repr_html_` was called, the string of the
schema.
+ if !self
+ .support_repr_html
+ .load(std::sync::atomic::Ordering::Relaxed)
+ {
+ if let Some((n, truncate)) = self.eager_eval(py)? {
+ return py
+ .detach(|| self.dataframe.show_string(n, truncate, false))
+ .to_pyerr();
+ }
}
let fields = self
.dtypes(py)?
@@ -2365,6 +2387,22 @@ impl PyDataFrame {
Ok(format!("DataFrame[{fields}]"))
}
+ /// The HTML table of the DataFrame, which notebooks show, as PySpark
Connect's
+ /// `_repr_html_`: with `spark.sql.repl.eagerEval.enabled`, the table of
the first
+ /// `maxNumRows` rows, values truncated to `truncate` characters, which
the server formats
+ /// (`HtmlString`); else None. Once called, `__repr__` is the string of
the schema.
+ fn _repr_html_(&self, py: Python<'_>) -> PyResult<Option<String>> {
+ self.support_repr_html
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ match self.eager_eval(py)? {
+ Some((n, truncate)) => py
+ .detach(|| self.dataframe.html_string(n, truncate))
+ .to_pyerr()
+ .map(Some),
+ None => Ok(None),
+ }
+ }
+
/// `DataFrame.mapInPandas` — map an iterator of pandas DataFrames, in a
stage of the
/// resource profile `profile`, if any, as PySpark's.
#[pyo3(name = "mapInPandas", signature = (func, schema, barrier = false,
profile = None))]
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index fc826d5..708c3bc 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -962,6 +962,25 @@ impl DataFrame {
}
}
+ /// The HTML table of the first `n` rows, values truncated to `truncate`
characters (0 for
+ /// none), as the server formats it. Mirrors the table of
`DataFrame._repr_html_()`.
+ pub fn html_string(&self, n: i32, truncate: i32) -> Result<String> {
+ let html = DataFrame::new(
+ self.session.clone(),
+ LogicalPlan::HtmlString {
+ input: Arc::clone(&self.plan),
+ num_rows: n,
+ truncate,
+ },
+ );
+ match html.collect()?.first().map(|row| row.values()) {
+ Some([Value::String(table)]) => Ok(table.clone()),
+ _ => Err(SparkError::connect_msg(
+ "HtmlString returned no table of one string",
+ )),
+ }
+ }
+
/// Print the first `n` rows as the server formats them, with values
truncated to 20
/// characters. Mirrors `DataFrame.show(n)`.
pub fn show(&self, n: usize) -> Result<()> {
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index bce9c66..c11521d 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -106,6 +106,13 @@ pub enum LogicalPlan {
truncate: i32,
vertical: bool,
},
+ /// HtmlString: the HTML table of `df._repr_html_()`, which the server
formats as one
+ /// string: `num_rows` rows, values truncated to `truncate` characters (0
for none).
+ HtmlString {
+ input: Arc<LogicalPlan>,
+ num_rows: i32,
+ truncate: i32,
+ },
/// Deduplicate: `df.distinct()` or `df.dropDuplicates()`.
Deduplicate {
input: Arc<LogicalPlan>,
@@ -854,6 +861,19 @@ impl LogicalPlan {
relation.rel_type =
Some(proto::relation::RelType::ShowString(Box::new(show)));
}
+ LogicalPlan::HtmlString {
+ input,
+ num_rows,
+ truncate,
+ } => {
+ let html = proto::HtmlString {
+ input: Some(Box::new(relation_of(input, plan_ids))),
+ num_rows: *num_rows,
+ truncate: *truncate,
+ };
+ relation.rel_type =
Some(proto::relation::RelType::HtmlString(Box::new(html)));
+ }
+
LogicalPlan::Deduplicate {
input,
all_columns_as_keys,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d7dc597..060e0cc 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1935,6 +1935,20 @@ def
test_sql_fails_at_a_statement_that_does_not_parse(session):
session.sql("SELEC 1")
+def test_repr_html_is_the_html_table_of_eager_evaluation_as_pyspark(session):
+ # As PySpark Connect's: _repr_html_ is None without eager evaluation, else
the HTML table of
+ # the DataFrame, after which __repr__ is the string of the schema.
+ df = session.range(2)
+ assert df._repr_html_() is None
+ session.conf.set("spark.sql.repl.eagerEval.enabled", "true")
+ try:
+ html = df._repr_html_()
+ assert html.startswith("<table border='1'>") and "<th>id</th>" in html
+ assert repr(df) == "DataFrame[id: bigint]"
+ assert "+---+" in repr(session.range(2))
+ finally:
+ session.conf.unset("spark.sql.repl.eagerEval.enabled")
+
def test_sql_formats_the_query_with_keyword_arguments_as_pyspark(session):
from decimal import Decimal
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 1f546c0..70dad79 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9458,6 +9458,38 @@ def _repr_struct(*fields):
return _repr_message(21, b"".join(_repr_message(1, _repr_field(name,
dtype)) for name, dtype in fields))
+def
test_repr_html_is_the_html_table_of_eager_evaluation_as_pyspark(mock_connect):
+ # As PySpark Connect's: _repr_html_ is None without eager evaluation, else
the HTML table
+ # that the server formats of an HtmlString relation of maxNumRows rows and
truncate; once
+ # it was called, __repr__ is the string of the schema.
+ import pyarrow as pa
+
+ spark, service = mock_connect
+ eager = {"spark.sql.repl.eagerEval.enabled": "false",
+ "spark.sql.repl.eagerEval.maxNumRows": "7",
+ "spark.sql.repl.eagerEval.truncate": "3"}
+ service.confs = eager
+ df = spark.range(1)
+ assert df._repr_html_() is None
+ service.confs = {**eager, "spark.sql.repl.eagerEval.enabled": "true"}
+ table = pa.table({"html_string": ["<table/>"]})
+ sink = pa.BufferOutputStream()
+ with pa.ipc.new_stream(sink, table.schema) as writer:
+ writer.write_table(table)
+ # ExecutePlanResponse.arrow_batch=2: ArrowBatch.row_count=1, data=2.
+ service.execute_payload = message_field(
+ 2, varint_field(1, 1) + message_field(2, sink.getvalue().to_pybytes()))
+ assert df._repr_html_() == "<table/>"
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.html_string=35:
HtmlString.num_rows=2,
+ # truncate=3.
+ plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+ html = parse_message(parse_message(plan[1][0])[35][0])
+ assert html[2] == [7] and html[3] == [3]
+ assert repr(df) == "DataFrame[id: bigint]"
+ # A DataFrame of no _repr_html_ shows the table of eager evaluation.
+ assert repr(spark.range(1)) == "<table/>"
+
+
@pytest.mark.parametrize("schema, expected", [
(_repr_struct(), "DataFrame[]"),
(_repr_struct(("id", _repr_message(7, b""))), "DataFrame[id: bigint]"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]