This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 0a4c3e8 [SPARK-61126][CONNECT][PYTHON] Make DataFrame.pandas_api take
index_col, and DataFrame.plot the PySparkPlotAccessor, in the Python client as
PySpark does
0a4c3e8 is described below
commit 0a4c3e8b973ca5172316e2071bb1edb0c1d03830
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 07:32:19 2026 +0900
[SPARK-61126][CONNECT][PYTHON] Make DataFrame.pandas_api take index_col,
and DataFrame.plot the PySparkPlotAccessor, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61126](https://issues.apache.org/jira/browse/SPARK-61126):
`DataFrame.pandas_api` and `DataFrame.plot` of the client are now those of
PySpark 4.3.0.
- **`pandas_api(index_col)`**: as PySpark's, it builds the `InternalFrame`
of the DataFrame. The index is of the columns of `index_col`
(`pyspark.pandas.namespace._get_index_map`). The result is the pandas-on-Spark
DataFrame of that frame.
- **`plot`**: as PySpark's, it is the `PySparkPlotAccessor` of
`pyspark.sql.plot`, of the DataFrame itself.
### Why are the changes needed?
- The client passed `index_col` to the constructor of
`pyspark.pandas.DataFrame`, which does not take it. So
`df.pandas_api(index_col="k")` raised `TypeError`.
- The client's `plot` was the plot accessor of its pandas-on-Spark
DataFrame, which differs from PySpark's accessor. Apache Spark's plot parity
tests failed on this client.
### Does this PR introduce _any_ user-facing change?
Yes:
- `pandas_api(index_col=...)` gives a pandas-on-Spark DataFrame indexed by
those columns.
- `df.plot` is PySpark's `PySparkPlotAccessor`, as with PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks that `df.plot` is a `PySparkPlotAccessor` of
the DataFrame itself.
- A new live test checks:
- the index and the columns of `pandas_api(index_col="k")`, and the index
of `pandas_api()`
- the figure of a line plot, if `plotly` is installed
- Apache Spark's plot parity tests (`pyspark/sql/tests/connect/plot/`) run
on this client against a local server, and all 18 pass. Before this change, 7
failed.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2181 passed, 8 skipped), and the
live drop-in and Arrow suites (177 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #324 from HyukjinKwon/SPARK-61126-pandas-api-plot.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 38 ++++++++++++++++++++++++++-----------
python/tests/test_dropin_e2e.py | 13 +++++++++++++
python/tests/test_dropin_offline.py | 11 +++++++++++
3 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 54033a3..4f59a5f 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2204,8 +2204,9 @@ impl PyDataFrame {
Ok(ei.unbind())
}
- /// Convert to a pandas-on-Spark DataFrame. Mirrors `DataFrame.pandas_api`;
- /// bridges into the vendored `pyspark.pandas`.
+ /// Convert to a pandas-on-Spark DataFrame, as PySpark's
`DataFrame.pandas_api`: of the
+ /// `InternalFrame` of this DataFrame, of the index of the columns of
`index_col`
+ /// (`pyspark.pandas.namespace._get_index_map`), of the vendored
`pyspark.pandas`.
#[pyo3(name = "pandas_api")]
#[pyo3(signature = (index_col=None))]
fn pandas_api(
@@ -2213,20 +2214,35 @@ impl PyDataFrame {
py: Python<'_>,
index_col: Option<Bound<'_, PyAny>>,
) -> PyResult<Py<PyAny>> {
- let ps = py.import("pyspark.pandas")?;
- let df_cls = ps.getattr("DataFrame")?;
+ let (index_spark_columns, index_names) = py
+ .import("pyspark.pandas.namespace")?
+ .getattr("_get_index_map")?
+ .call1((&slf, index_col))?
+ .extract::<(Bound<'_, PyAny>, Bound<'_, PyAny>)>()?;
let kwargs = pyo3::types::PyDict::new(py);
- if let Some(ic) = index_col {
- kwargs.set_item("index_col", ic)?;
- }
- Ok(df_cls.call((slf,), Some(&kwargs))?.unbind())
+ kwargs.set_item("spark_frame", &slf)?;
+ kwargs.set_item("index_spark_columns", index_spark_columns)?;
+ kwargs.set_item("index_names", index_names)?;
+ let internal = py
+ .import("pyspark.pandas.internal")?
+ .getattr("InternalFrame")?
+ .call((), Some(&kwargs))?;
+ Ok(py
+ .import("pyspark.pandas.frame")?
+ .getattr("DataFrame")?
+ .call1((internal,))?
+ .unbind())
}
- /// pandas-on-Spark plotting accessor. Mirrors `DataFrame.plot`.
+ /// The plot accessor of the DataFrame, as PySpark's `DataFrame.plot`: the
+ /// `PySparkPlotAccessor` of `pyspark.sql.plot`, of this DataFrame.
#[getter]
fn plot(slf: Bound<'_, Self>, py: Python<'_>) -> PyResult<Py<PyAny>> {
- let psdf = Self::pandas_api(slf, py, None)?;
- Ok(psdf.bind(py).getattr("plot")?.unbind())
+ Ok(py
+ .import("pyspark.sql.plot")?
+ .getattr("PySparkPlotAccessor")?
+ .call1((slf,))?
+ .unbind())
}
/// The schema of this DataFrame as a `StructType` (a property, mirroring
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 9e0391c..ba4bcbe 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -70,6 +70,19 @@ def
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
assert list(rows) == []
+def test_pandas_api_takes_its_index_and_plot_is_of_pyspark(session):
+ # As PySpark's DataFrame.pandas_api(index_col): a pandas-on-Spark
DataFrame of the index
+ # of the columns of index_col; and DataFrame.plot, the PySparkPlotAccessor
of the DataFrame.
+ sdf = session.createDataFrame([("a", 1, 2.0), ("b", 2, 3.0)], ["k", "i",
"f"])
+ psdf = sdf.pandas_api(index_col="k")
+ assert psdf.index.name == "k" and list(psdf.columns) == ["i", "f"]
+ assert psdf.sort_index().to_pandas()["i"].tolist() == [1, 2]
+ assert sdf.pandas_api().index.name is None
+ pytest.importorskip("plotly")
+ figure = sdf.plot.line(x="k", y=["i", "f"])
+ assert type(figure).__name__ == "Figure" and len(figure.data) == 2
+
+
def test_udtfs_of_ddl_return_types_are_unparsed_as_pyspark(session):
# As PySpark Connect's: the returnType of a UDTF of a DDL string is an
UnparsedDataType,
# which the server parses.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 397b587..128059b 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -482,6 +482,17 @@ def test_storage_level():
assert StorageLevel.DISK_ONLY != StorageLevel.MEMORY_ONLY
+def test_dataframe_plot_is_the_plot_accessor_of_pyspark():
+ # As PySpark's DataFrame.plot: the PySparkPlotAccessor of
pyspark.sql.plot, of the
+ # DataFrame itself.
+ from pyspark.sql.plot import PySparkPlotAccessor
+
+ df = _offline_session().range(1)
+ plot = df.plot
+ assert type(plot) is PySparkPlotAccessor
+ assert plot.data is df
+
+
def
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect):
# As PySpark's, checkpoint(eager) and localCheckpoint(eager, storageLevel)
send them in
# their CheckpointCommand, of which the mock answers with a
CachedRemoteRelation.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]