This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 1096801ead29 [SPARK-47828][CONNECT][PYTHON] DataFrameWriterV2.overwrite` fails with invalid plan 1096801ead29 is described below commit 1096801ead29a8b62e915a0348b8adaad060b215 Author: Ruifeng Zheng <ruife...@apache.org> AuthorDate: Mon Apr 15 12:21:01 2024 +0800 [SPARK-47828][CONNECT][PYTHON] DataFrameWriterV2.overwrite` fails with invalid plan ### What changes were proposed in this pull request? `DataFrameWriterV2.overwrite` fails with invalid plan ### Why are the changes needed? bug fix ### Does this PR introduce _any_ user-facing change? yes ### How was this patch tested? added test ### Was this patch authored or co-authored using generative AI tooling? no Closes #46023 from zhengruifeng/connect_fix_overwrite. Authored-by: Ruifeng Zheng <ruife...@apache.org> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- python/pyspark/sql/connect/plan.py | 8 ++++---- python/pyspark/sql/tests/test_readwriter.py | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index 72b8372c8039..239ee23c2061 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -1751,16 +1751,16 @@ class WriteOperationV2(LogicalPlan): plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_CREATE elif wm == "overwrite": plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_OVERWRITE + if self.overwrite_condition is not None: + plan.write_operation_v2.overwrite_condition.CopyFrom( + self.overwrite_condition.to_plan(session) + ) elif wm == "overwrite_partitions": plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_OVERWRITE_PARTITIONS elif wm == "append": plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_APPEND elif wm == "replace": plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_REPLACE - if self.overwrite_condition is not None: - plan.write_operation_v2.overwrite_condition.CopyFrom( - self.overwrite_condition.to_plan(session) - ) elif wm == "create_or_replace": plan.write_operation_v2.mode = proto.WriteOperationV2.Mode.MODE_CREATE_OR_REPLACE else: diff --git a/python/pyspark/sql/tests/test_readwriter.py b/python/pyspark/sql/tests/test_readwriter.py index 85057f37a181..5784d2c72973 100644 --- a/python/pyspark/sql/tests/test_readwriter.py +++ b/python/pyspark/sql/tests/test_readwriter.py @@ -252,6 +252,11 @@ class ReadwriterV2TestsMixin: ): df.writeTo("test_table").create() + def test_table_overwrite(self): + df = self.df + with self.assertRaisesRegex(AnalysisException, "TABLE_OR_VIEW_NOT_FOUND"): + df.writeTo("test_table").overwrite(lit(True)) + class ReadwriterTests(ReadwriterTestsMixin, ReusedSQLTestCase): pass --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org