This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9241195  [SPARK-61053][CONNECT][PYTHON] Take the arguments of 
overwrite, partitionedBy and tableProperty of DataFrameWriterV2 in the Python 
client as PySpark does
9241195 is described below

commit 9241195f2890e7af2ec461e09289753c7564d260
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:52:33 2026 +0900

    [SPARK-61053][CONNECT][PYTHON] Take the arguments of overwrite, 
partitionedBy and tableProperty of DataFrameWriterV2 in the Python client as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61053](https://issues.apache.org/jira/browse/SPARK-61053): 
`DataFrameWriterV2` now takes the arguments of `overwrite`, `partitionedBy` and 
`tableProperty` as PySpark Connect's does.
    
    - `overwrite(condition)` takes a Column or a column name, as `F._to_col` 
does.
    - `partitionedBy(col, *cols)` takes a Column or a column name for each 
argument. It raises `NOT_EXPECTED_TYPE` of `col` for any other argument, also 
for a list.
    - `tableProperty(property, None)` leaves the property unset, also after an 
earlier value.
    
    The core adds `DataFrameWriterV2::unset_table_property`.
    
    ### Why are the changes needed?
    
    The client:
    - raised a `TypeError` for a str condition of `overwrite`
    - took a list of columns in `partitionedBy`, and named the argument of its 
error `cols`
    - raised a `TypeError` for a property of None
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods take the arguments of PySpark 4.3.0 and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test checks that a property of None is not set on the table, 
the `NOT_EXPECTED_TYPE` of a list in `partitionedBy`, and that a column name in 
`overwrite` reaches the server. It passes with official PySpark 4.3.0, and 
fails without this change.
    - A new offline test checks the `WriteOperationV2`s that these send to a 
mock Connect server, and their errors.
    - The lib tests of both crates (574 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the 
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #251 from HyukjinKwon/SPARK-61053-writer-v2-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs     | 27 +++++++++++++---------
 crates/spark-connect/src/readwriter.rs |  7 ++++++
 python/tests/test_dropin_e2e.py        | 26 +++++++++++++++++++++
 python/tests/test_dropin_offline.py    | 41 ++++++++++++++++++++++++++++++++++
 4 files changed, 91 insertions(+), 10 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index bda68b4..23e3ca9 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3244,25 +3244,32 @@ impl PyDataFrameWriterV2 {
         Ok(slf)
     }
 
+    /// Set the table property `property` to `value`, or leave it unset for 
None, as PySpark's.
     #[pyo3(name = "tableProperty")]
     fn table_property<'py>(
         mut slf: PyRefMut<'py, Self>,
         property: &str,
-        value: &str,
+        value: Option<&str>,
     ) -> PyResult<PyRefMut<'py, Self>> {
-        slf.inner = slf.inner.clone().table_property(property, value);
+        slf.inner = match value {
+            Some(value) => slf.inner.clone().table_property(property, value),
+            None => slf.inner.clone().unset_table_property(property),
+        };
         Ok(slf)
     }
 
+    /// Partition by the columns, each a Column or a column name, as PySpark's 
`F._to_col` of
+    /// each, else `NOT_EXPECTED_TYPE` of `col`, also for a list.
     #[pyo3(name = "partitionedBy", signature = (col, *cols))]
     fn partitioned_by<'py>(
         mut slf: PyRefMut<'py, Self>,
         col: Bound<'_, PyAny>,
         cols: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyRefMut<'py, Self>> {
-        let mut all = vec![col];
-        all.extend(cols);
-        let columns = to_column_list(all)?;
+        let columns = std::iter::once(&col)
+            .chain(&cols)
+            .map(column_or_name)
+            .collect::<PyResult<Vec<_>>>()?;
         slf.inner = slf.inner.clone().partition_by(columns);
         Ok(slf)
     }
@@ -3296,11 +3303,11 @@ impl PyDataFrameWriterV2 {
         self.inner.clone().append().to_pyerr()
     }
 
-    fn overwrite(&mut self, condition: &PyColumn) -> PyResult<()> {
-        self.inner
-            .clone()
-            .overwrite(condition.column.clone())
-            .to_pyerr()
+    /// Overwrite the rows of `condition`, a Column or a column name, as 
PySpark's `F._to_col`,
+    /// else `NOT_EXPECTED_TYPE` of `col`.
+    fn overwrite(&mut self, condition: &Bound<'_, PyAny>) -> PyResult<()> {
+        let condition = column_or_name(condition)?;
+        self.inner.clone().overwrite(condition).to_pyerr()
     }
 
     #[pyo3(name = "overwritePartitions")]
diff --git a/crates/spark-connect/src/readwriter.rs 
b/crates/spark-connect/src/readwriter.rs
index fd08538..3ed5044 100644
--- a/crates/spark-connect/src/readwriter.rs
+++ b/crates/spark-connect/src/readwriter.rs
@@ -554,6 +554,13 @@ impl DataFrameWriterV2 {
         self
     }
 
+    /// Leave the table property `property` unset, as PySpark's 
`tableProperty(property, None)`,
+    /// which the command does not send.
+    pub fn unset_table_property(mut self, property: &str) -> Self {
+        self.table_properties.remove(property);
+        self
+    }
+
     /// Partition the output table by the given columns.
     pub fn partition_by<C: Into<Column>>(mut self, columns: impl 
IntoIterator<Item = C>) -> Self {
         self.partition_cols = columns.into_iter().map(Into::into).collect();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d499e05..1db0973 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -967,6 +967,32 @@ def 
test_dataframe_columns_resolve_against_their_dataframe_as_pyspark(session):
     assert error.value.getCondition() == "AMBIGUOUS_COLUMN_REFERENCE"
 
 
+def test_writer_v2_takes_its_arguments_as_pyspark(session):
+    # As PySpark's DataFrameWriterV2: tableProperty(property, None) leaves the 
property unset,
+    # partitionedBy raises NOT_EXPECTED_TYPE of col for a list, and overwrite 
takes a column
+    # name, which the server takes: a table of the session catalog does not 
overwrite by a
+    # condition.
+    import uuid
+
+    from pyspark.errors import AnalysisException, PySparkTypeError
+
+    name = f"writer_v2_arguments_{uuid.uuid4().hex[:8]}"
+    df = session.createDataFrame([(1, True), (2, False)], "id int, flag 
boolean")
+    try:
+        writer = df.writeTo(name).using("parquet").tableProperty("a", "1")
+        writer.tableProperty("a", None).tableProperty("b", "2").create()
+        rows = session.sql(f"SHOW TBLPROPERTIES {name}").collect()
+        properties = {row.key: row.value for row in rows}
+        assert properties.get("b") == "2" and "a" not in properties
+        with pytest.raises(PySparkTypeError) as e:
+            df.writeTo(name).partitionedBy(["id"])
+        assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+        with pytest.raises(AnalysisException):
+            df.writeTo(name).overwrite("flag")
+    finally:
+        session.sql(f"DROP TABLE IF EXISTS {name}")
+
+
 def test_collect_reads_a_result_across_stream_ends(session):
     # Takes several response streams when the server's senderMaxStreamSize is 
small.
     n = 300_000
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 8f8e92e..f2a04b8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9211,6 +9211,47 @@ def 
test_writer_setters_mutate_and_return_self(offline_spark):
     assert w.mode("append") is w
 
 
+def test_writer_v2_takes_its_arguments_as_pyspark(mock_connect):
+    # As PySpark's DataFrameWriterV2: overwrite takes a Column or a column 
name, and
+    # partitionedBy a Column or a column name of each argument, else 
NOT_EXPECTED_TYPE of col,
+    # also for a list; tableProperty(property, None) leaves the property unset.
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    df = spark.range(1)
+
+    def sent():
+        # ExecutePlanRequest.plan=3, Plan.command=2, 
Command.write_operation_v2=4.
+        plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+        return parse_message(parse_message(plan[2][0])[4][0])
+
+    df.writeTo("t").overwrite("flag")
+    # WriteOperationV2.overwrite_condition=8: 
Expression.unresolved_attribute=2 of
+    # UnresolvedAttribute.unparsed_identifier=1.
+    condition = parse_message(sent()[8][0])
+    assert parse_message(condition[2][0])[1] == [b"flag"]
+    writer = df.writeTo("t").tableProperty("a", "1").tableProperty("b", "2")
+    writer.tableProperty("a", None).append()
+    # WriteOperationV2.table_properties=6, of entries of key=1 and value=2.
+    assert [parse_message(entry)[1] for entry in sent().get(6, [])] == [[b"b"]]
+    for args, arg_type in [((["id"],), "list"), (("id", 1), "int")]:
+        with pytest.raises(PySparkTypeError) as e:
+            df.writeTo("t").partitionedBy(*args)
+        assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert e.value.getMessageParameters() == {
+            "arg_name": "col",
+            "expected_type": "Column or str",
+            "arg_type": arg_type,
+        }
+    with pytest.raises(PySparkTypeError) as e:
+        df.writeTo("t").overwrite(1)
+    assert e.value.getMessageParameters()["arg_name"] == "col"
+    df.writeTo("t").partitionedBy("id", F.col("id")).append()
+    # WriteOperationV2.partitioning_columns=4.
+    assert len(sent()[4]) == 2
+
+
 def test_writer_v2_setters_mutate_and_return_self(offline_spark):
     from pyspark.sql import functions as F
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to