This is an automated email from the ASF dual-hosted git repository.
beliefer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f5cf3101991 [SPARK-45412][PYTHON][CONNECT][FOLLOW-UP] Remove
unnecessary check
f5cf3101991 is described below
commit f5cf310199132f62b779e0244d15f7680e2ba856
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Oct 9 18:07:59 2023 +0800
[SPARK-45412][PYTHON][CONNECT][FOLLOW-UP] Remove unnecessary check
### What changes were proposed in this pull request?
Remove unnecessary check
### Why are the changes needed?
https://github.com/apache/spark/pull/43215 already validates the plan in
`__init__`
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #43287 from zhengruifeng/SPARK-45412-followup.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Jiaan Geng <[email protected]>
---
python/pyspark/sql/connect/dataframe.py | 16 +---------------
1 file changed, 1 insertion(+), 15 deletions(-)
diff --git a/python/pyspark/sql/connect/dataframe.py
b/python/pyspark/sql/connect/dataframe.py
index 2c0a75fad46..4044fab3bb3 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -169,7 +169,6 @@ class DataFrame:
@property
def write(self) -> "DataFrameWriter":
- assert self._plan is not None
return DataFrameWriter(self._plan, self._session)
write.__doc__ = PySparkDataFrame.write.__doc__
@@ -1096,11 +1095,6 @@ class DataFrame:
union.__doc__ = PySparkDataFrame.union.__doc__
def unionAll(self, other: "DataFrame") -> "DataFrame":
- if other._plan is None:
- raise PySparkValueError(
- error_class="MISSING_VALID_PLAN",
- message_parameters={"operator": "Union"},
- )
self._check_same_session(other)
return DataFrame.withPlan(
plan.SetOperation(self._plan, other._plan, "union", is_all=True),
session=self._session
@@ -2030,8 +2024,6 @@ class DataFrame:
mapInArrow.__doc__ = PySparkDataFrame.mapInArrow.__doc__
def foreach(self, f: Callable[[Row], None]) -> None:
- assert self._plan is not None
-
def foreach_func(row: Any) -> None:
f(row)
@@ -2042,8 +2034,6 @@ class DataFrame:
foreach.__doc__ = PySparkDataFrame.foreach.__doc__
def foreachPartition(self, f: Callable[[Iterator[Row]], None]) -> None:
- assert self._plan is not None
-
schema = self.schema
field_converters = [
ArrowTableToRowsConversion._create_converter(f.dataType) for f in
schema.fields
@@ -2069,14 +2059,12 @@ class DataFrame:
@property
def writeStream(self) -> DataStreamWriter:
- assert self._plan is not None
return DataStreamWriter(plan=self._plan, session=self._session)
writeStream.__doc__ = PySparkDataFrame.writeStream.__doc__
def sameSemantics(self, other: "DataFrame") -> bool:
- assert self._plan is not None
- assert other._plan is not None
+ self._check_same_session(other)
return self._session.client.same_semantics(
plan=self._plan.to_proto(self._session.client),
other=other._plan.to_proto(other._session.client),
@@ -2085,7 +2073,6 @@ class DataFrame:
sameSemantics.__doc__ = PySparkDataFrame.sameSemantics.__doc__
def semanticHash(self) -> int:
- assert self._plan is not None
return self._session.client.semantic_hash(
plan=self._plan.to_proto(self._session.client),
)
@@ -2093,7 +2080,6 @@ class DataFrame:
semanticHash.__doc__ = PySparkDataFrame.semanticHash.__doc__
def writeTo(self, table: str) -> "DataFrameWriterV2":
- assert self._plan is not None
return DataFrameWriterV2(self._plan, self._session, table)
writeTo.__doc__ = PySparkDataFrame.writeTo.__doc__
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
