This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9241195 [SPARK-61053][CONNECT][PYTHON] Take the arguments of
overwrite, partitionedBy and tableProperty of DataFrameWriterV2 in the Python
client as PySpark does
9241195 is described below
commit 9241195f2890e7af2ec461e09289753c7564d260
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:52:33 2026 +0900
[SPARK-61053][CONNECT][PYTHON] Take the arguments of overwrite,
partitionedBy and tableProperty of DataFrameWriterV2 in the Python client as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61053](https://issues.apache.org/jira/browse/SPARK-61053):
`DataFrameWriterV2` now takes the arguments of `overwrite`, `partitionedBy` and
`tableProperty` as PySpark Connect's does.
- `overwrite(condition)` takes a Column or a column name, as `F._to_col`
does.
- `partitionedBy(col, *cols)` takes a Column or a column name for each
argument. It raises `NOT_EXPECTED_TYPE` of `col` for any other argument, also
for a list.
- `tableProperty(property, None)` leaves the property unset, also after an
earlier value.
The core adds `DataFrameWriterV2::unset_table_property`.
### Why are the changes needed?
The client:
- raised a `TypeError` for a str condition of `overwrite`
- took a list of columns in `partitionedBy`, and named the argument of its
error `cols`
- raised a `TypeError` for a property of None
### Does this PR introduce _any_ user-facing change?
Yes. These methods take the arguments of PySpark 4.3.0 and raise its errors.
### How was this patch tested?
- A new live test checks that a property of None is not set on the table,
the `NOT_EXPECTED_TYPE` of a list in `partitionedBy`, and that a column name in
`overwrite` reaches the server. It passes with official PySpark 4.3.0, and
fails without this change.
- A new offline test checks the `WriteOperationV2`s that these send to a
mock Connect server, and their errors.
- The lib tests of both crates (574 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #251 from HyukjinKwon/SPARK-61053-writer-v2-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 27 +++++++++++++---------
crates/spark-connect/src/readwriter.rs | 7 ++++++
python/tests/test_dropin_e2e.py | 26 +++++++++++++++++++++
python/tests/test_dropin_offline.py | 41 ++++++++++++++++++++++++++++++++++
4 files changed, 91 insertions(+), 10 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index bda68b4..23e3ca9 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -3244,25 +3244,32 @@ impl PyDataFrameWriterV2 {
Ok(slf)
}
+ /// Set the table property `property` to `value`, or leave it unset for
None, as PySpark's.
#[pyo3(name = "tableProperty")]
fn table_property<'py>(
mut slf: PyRefMut<'py, Self>,
property: &str,
- value: &str,
+ value: Option<&str>,
) -> PyResult<PyRefMut<'py, Self>> {
- slf.inner = slf.inner.clone().table_property(property, value);
+ slf.inner = match value {
+ Some(value) => slf.inner.clone().table_property(property, value),
+ None => slf.inner.clone().unset_table_property(property),
+ };
Ok(slf)
}
+ /// Partition by the columns, each a Column or a column name, as PySpark's
`F._to_col` of
+ /// each, else `NOT_EXPECTED_TYPE` of `col`, also for a list.
#[pyo3(name = "partitionedBy", signature = (col, *cols))]
fn partitioned_by<'py>(
mut slf: PyRefMut<'py, Self>,
col: Bound<'_, PyAny>,
cols: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyRefMut<'py, Self>> {
- let mut all = vec![col];
- all.extend(cols);
- let columns = to_column_list(all)?;
+ let columns = std::iter::once(&col)
+ .chain(&cols)
+ .map(column_or_name)
+ .collect::<PyResult<Vec<_>>>()?;
slf.inner = slf.inner.clone().partition_by(columns);
Ok(slf)
}
@@ -3296,11 +3303,11 @@ impl PyDataFrameWriterV2 {
self.inner.clone().append().to_pyerr()
}
- fn overwrite(&mut self, condition: &PyColumn) -> PyResult<()> {
- self.inner
- .clone()
- .overwrite(condition.column.clone())
- .to_pyerr()
+ /// Overwrite the rows of `condition`, a Column or a column name, as
PySpark's `F._to_col`,
+ /// else `NOT_EXPECTED_TYPE` of `col`.
+ fn overwrite(&mut self, condition: &Bound<'_, PyAny>) -> PyResult<()> {
+ let condition = column_or_name(condition)?;
+ self.inner.clone().overwrite(condition).to_pyerr()
}
#[pyo3(name = "overwritePartitions")]
diff --git a/crates/spark-connect/src/readwriter.rs
b/crates/spark-connect/src/readwriter.rs
index fd08538..3ed5044 100644
--- a/crates/spark-connect/src/readwriter.rs
+++ b/crates/spark-connect/src/readwriter.rs
@@ -554,6 +554,13 @@ impl DataFrameWriterV2 {
self
}
+ /// Leave the table property `property` unset, as PySpark's
`tableProperty(property, None)`,
+ /// which the command does not send.
+ pub fn unset_table_property(mut self, property: &str) -> Self {
+ self.table_properties.remove(property);
+ self
+ }
+
/// Partition the output table by the given columns.
pub fn partition_by<C: Into<Column>>(mut self, columns: impl
IntoIterator<Item = C>) -> Self {
self.partition_cols = columns.into_iter().map(Into::into).collect();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d499e05..1db0973 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -967,6 +967,32 @@ def
test_dataframe_columns_resolve_against_their_dataframe_as_pyspark(session):
assert error.value.getCondition() == "AMBIGUOUS_COLUMN_REFERENCE"
+def test_writer_v2_takes_its_arguments_as_pyspark(session):
+ # As PySpark's DataFrameWriterV2: tableProperty(property, None) leaves the
property unset,
+ # partitionedBy raises NOT_EXPECTED_TYPE of col for a list, and overwrite
takes a column
+ # name, which the server takes: a table of the session catalog does not
overwrite by a
+ # condition.
+ import uuid
+
+ from pyspark.errors import AnalysisException, PySparkTypeError
+
+ name = f"writer_v2_arguments_{uuid.uuid4().hex[:8]}"
+ df = session.createDataFrame([(1, True), (2, False)], "id int, flag
boolean")
+ try:
+ writer = df.writeTo(name).using("parquet").tableProperty("a", "1")
+ writer.tableProperty("a", None).tableProperty("b", "2").create()
+ rows = session.sql(f"SHOW TBLPROPERTIES {name}").collect()
+ properties = {row.key: row.value for row in rows}
+ assert properties.get("b") == "2" and "a" not in properties
+ with pytest.raises(PySparkTypeError) as e:
+ df.writeTo(name).partitionedBy(["id"])
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ with pytest.raises(AnalysisException):
+ df.writeTo(name).overwrite("flag")
+ finally:
+ session.sql(f"DROP TABLE IF EXISTS {name}")
+
+
def test_collect_reads_a_result_across_stream_ends(session):
# Takes several response streams when the server's senderMaxStreamSize is
small.
n = 300_000
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 8f8e92e..f2a04b8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9211,6 +9211,47 @@ def
test_writer_setters_mutate_and_return_self(offline_spark):
assert w.mode("append") is w
+def test_writer_v2_takes_its_arguments_as_pyspark(mock_connect):
+ # As PySpark's DataFrameWriterV2: overwrite takes a Column or a column
name, and
+ # partitionedBy a Column or a column name of each argument, else
NOT_EXPECTED_TYPE of col,
+ # also for a list; tableProperty(property, None) leaves the property unset.
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ df = spark.range(1)
+
+ def sent():
+ # ExecutePlanRequest.plan=3, Plan.command=2,
Command.write_operation_v2=4.
+ plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+ return parse_message(parse_message(plan[2][0])[4][0])
+
+ df.writeTo("t").overwrite("flag")
+ # WriteOperationV2.overwrite_condition=8:
Expression.unresolved_attribute=2 of
+ # UnresolvedAttribute.unparsed_identifier=1.
+ condition = parse_message(sent()[8][0])
+ assert parse_message(condition[2][0])[1] == [b"flag"]
+ writer = df.writeTo("t").tableProperty("a", "1").tableProperty("b", "2")
+ writer.tableProperty("a", None).append()
+ # WriteOperationV2.table_properties=6, of entries of key=1 and value=2.
+ assert [parse_message(entry)[1] for entry in sent().get(6, [])] == [[b"b"]]
+ for args, arg_type in [((["id"],), "list"), (("id", 1), "int")]:
+ with pytest.raises(PySparkTypeError) as e:
+ df.writeTo("t").partitionedBy(*args)
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ assert e.value.getMessageParameters() == {
+ "arg_name": "col",
+ "expected_type": "Column or str",
+ "arg_type": arg_type,
+ }
+ with pytest.raises(PySparkTypeError) as e:
+ df.writeTo("t").overwrite(1)
+ assert e.value.getMessageParameters()["arg_name"] == "col"
+ df.writeTo("t").partitionedBy("id", F.col("id")).append()
+ # WriteOperationV2.partitioning_columns=4.
+ assert len(sent()[4]) == 2
+
+
def test_writer_v2_setters_mutate_and_return_self(offline_spark):
from pyspark.sql import functions as F
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]