This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a4c3e8  [SPARK-61126][CONNECT][PYTHON] Make DataFrame.pandas_api take 
index_col, and DataFrame.plot the PySparkPlotAccessor, in the Python client as 
PySpark does
0a4c3e8 is described below

commit 0a4c3e8b973ca5172316e2071bb1edb0c1d03830
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 07:32:19 2026 +0900

    [SPARK-61126][CONNECT][PYTHON] Make DataFrame.pandas_api take index_col, 
and DataFrame.plot the PySparkPlotAccessor, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61126](https://issues.apache.org/jira/browse/SPARK-61126): 
`DataFrame.pandas_api` and `DataFrame.plot` of the client are now those of 
PySpark 4.3.0.
    
    - **`pandas_api(index_col)`**: as PySpark's, it builds the `InternalFrame` 
of the DataFrame. The index is of the columns of `index_col` 
(`pyspark.pandas.namespace._get_index_map`). The result is the pandas-on-Spark 
DataFrame of that frame.
    - **`plot`**: as PySpark's, it is the `PySparkPlotAccessor` of 
`pyspark.sql.plot`, of the DataFrame itself.
    
    ### Why are the changes needed?
    
    - The client passed `index_col` to the constructor of 
`pyspark.pandas.DataFrame`, which does not take it. So 
`df.pandas_api(index_col="k")` raised `TypeError`.
    - The client's `plot` was the plot accessor of its pandas-on-Spark 
DataFrame, which differs from PySpark's accessor. Apache Spark's plot parity 
tests failed on this client.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes:
    - `pandas_api(index_col=...)` gives a pandas-on-Spark DataFrame indexed by 
those columns.
    - `df.plot` is PySpark's `PySparkPlotAccessor`, as with PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks that `df.plot` is a `PySparkPlotAccessor` of 
the DataFrame itself.
    - A new live test checks:
      - the index and the columns of `pandas_api(index_col="k")`, and the index 
of `pandas_api()`
      - the figure of a line plot, if `plotly` is installed
    - Apache Spark's plot parity tests (`pyspark/sql/tests/connect/plot/`) run 
on this client against a local server, and all 18 pass. Before this change, 7 
failed.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2181 passed, 8 skipped), and the 
live drop-in and Arrow suites (177 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #324 from HyukjinKwon/SPARK-61126-pandas-api-plot.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 38 ++++++++++++++++++++++++++-----------
 python/tests/test_dropin_e2e.py     | 13 +++++++++++++
 python/tests/test_dropin_offline.py | 11 +++++++++++
 3 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 54033a3..4f59a5f 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2204,8 +2204,9 @@ impl PyDataFrame {
         Ok(ei.unbind())
     }
 
-    /// Convert to a pandas-on-Spark DataFrame. Mirrors `DataFrame.pandas_api`;
-    /// bridges into the vendored `pyspark.pandas`.
+    /// Convert to a pandas-on-Spark DataFrame, as PySpark's 
`DataFrame.pandas_api`: of the
+    /// `InternalFrame` of this DataFrame, of the index of the columns of 
`index_col`
+    /// (`pyspark.pandas.namespace._get_index_map`), of the vendored 
`pyspark.pandas`.
     #[pyo3(name = "pandas_api")]
     #[pyo3(signature = (index_col=None))]
     fn pandas_api(
@@ -2213,20 +2214,35 @@ impl PyDataFrame {
         py: Python<'_>,
         index_col: Option<Bound<'_, PyAny>>,
     ) -> PyResult<Py<PyAny>> {
-        let ps = py.import("pyspark.pandas")?;
-        let df_cls = ps.getattr("DataFrame")?;
+        let (index_spark_columns, index_names) = py
+            .import("pyspark.pandas.namespace")?
+            .getattr("_get_index_map")?
+            .call1((&slf, index_col))?
+            .extract::<(Bound<'_, PyAny>, Bound<'_, PyAny>)>()?;
         let kwargs = pyo3::types::PyDict::new(py);
-        if let Some(ic) = index_col {
-            kwargs.set_item("index_col", ic)?;
-        }
-        Ok(df_cls.call((slf,), Some(&kwargs))?.unbind())
+        kwargs.set_item("spark_frame", &slf)?;
+        kwargs.set_item("index_spark_columns", index_spark_columns)?;
+        kwargs.set_item("index_names", index_names)?;
+        let internal = py
+            .import("pyspark.pandas.internal")?
+            .getattr("InternalFrame")?
+            .call((), Some(&kwargs))?;
+        Ok(py
+            .import("pyspark.pandas.frame")?
+            .getattr("DataFrame")?
+            .call1((internal,))?
+            .unbind())
     }
 
-    /// pandas-on-Spark plotting accessor. Mirrors `DataFrame.plot`.
+    /// The plot accessor of the DataFrame, as PySpark's `DataFrame.plot`: the
+    /// `PySparkPlotAccessor` of `pyspark.sql.plot`, of this DataFrame.
     #[getter]
     fn plot(slf: Bound<'_, Self>, py: Python<'_>) -> PyResult<Py<PyAny>> {
-        let psdf = Self::pandas_api(slf, py, None)?;
-        Ok(psdf.bind(py).getattr("plot")?.unbind())
+        Ok(py
+            .import("pyspark.sql.plot")?
+            .getattr("PySparkPlotAccessor")?
+            .call1((slf,))?
+            .unbind())
     }
 
     /// The schema of this DataFrame as a `StructType` (a property, mirroring
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 9e0391c..ba4bcbe 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -70,6 +70,19 @@ def 
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
     assert list(rows) == []
 
 
+def test_pandas_api_takes_its_index_and_plot_is_of_pyspark(session):
+    # As PySpark's DataFrame.pandas_api(index_col): a pandas-on-Spark 
DataFrame of the index
+    # of the columns of index_col; and DataFrame.plot, the PySparkPlotAccessor 
of the DataFrame.
+    sdf = session.createDataFrame([("a", 1, 2.0), ("b", 2, 3.0)], ["k", "i", 
"f"])
+    psdf = sdf.pandas_api(index_col="k")
+    assert psdf.index.name == "k" and list(psdf.columns) == ["i", "f"]
+    assert psdf.sort_index().to_pandas()["i"].tolist() == [1, 2]
+    assert sdf.pandas_api().index.name is None
+    pytest.importorskip("plotly")
+    figure = sdf.plot.line(x="k", y=["i", "f"])
+    assert type(figure).__name__ == "Figure" and len(figure.data) == 2
+
+
 def test_udtfs_of_ddl_return_types_are_unparsed_as_pyspark(session):
     # As PySpark Connect's: the returnType of a UDTF of a DDL string is an 
UnparsedDataType,
     # which the server parses.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 397b587..128059b 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -482,6 +482,17 @@ def test_storage_level():
     assert StorageLevel.DISK_ONLY != StorageLevel.MEMORY_ONLY
 
 
+def test_dataframe_plot_is_the_plot_accessor_of_pyspark():
+    # As PySpark's DataFrame.plot: the PySparkPlotAccessor of 
pyspark.sql.plot, of the
+    # DataFrame itself.
+    from pyspark.sql.plot import PySparkPlotAccessor
+
+    df = _offline_session().range(1)
+    plot = df.plot
+    assert type(plot) is PySparkPlotAccessor
+    assert plot.data is df
+
+
 def 
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect):
     # As PySpark's, checkpoint(eager) and localCheckpoint(eager, storageLevel) 
send them in
     # their CheckpointCommand, of which the mock answers with a 
CachedRemoteRelation.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to