zhengruifeng commented on code in PR #38506:
URL: https://github.com/apache/spark/pull/38506#discussion_r1017492795
##########
python/pyspark/sql/connect/dataframe.py:
##########
@@ -317,7 +319,83 @@ def unionByName(self, other: "DataFrame",
allowMissingColumns: bool = False) ->
if other._plan is None:
raise ValueError("Argument to UnionByName does not contain a valid
plan.")
return DataFrame.withPlan(
- plan.UnionAll(self._plan, other._plan, allowMissingColumns),
session=self._session
+ plan.SetOperation(
+ self._plan, other._plan, "union", is_all=True,
by_name=allowMissingColumns
+ ),
+ session=self._session,
+ )
+
+ def exceptAll(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing rows in this
:class:`DataFrame` but
+ not in another :class:`DataFrame` while preserving duplicates.
+
+ This is equivalent to `EXCEPT ALL` in SQL.
+ As standard in SQL, this function resolves columns by position (not by
name).
+
+ .. versionadded:: 2.4.0
+
+ Parameters
+ ----------
+ other : :class:`DataFrame`
+ The other :class:`DataFrame` to compare to.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ """
+ return DataFrame.withPlan(
+ plan.SetOperation(self._plan, other._plan, "except", is_all=True),
session=self._session
+ )
+
+ def intersect(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing rows only in
+ both this :class:`DataFrame` and another :class:`DataFrame`.
+ Note that any duplicates are removed. To preserve duplicates
+ use :func:`intersectAll`.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ other : :class:`DataFrame`
+ Another :class:`DataFrame` that needs to be combined.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ Combined DataFrame.
+
+ Notes
+ -----
+ This is equivalent to `INTERSECT` in SQL.
+ """
+ return DataFrame.withPlan(
+ plan.SetOperation(self._plan, other._plan, "intersect",
is_all=False),
+ session=self._session,
+ )
+
+ def intersectAll(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing rows in both this
:class:`DataFrame`
+ and another :class:`DataFrame` while preserving duplicates.
+
+ This is equivalent to `INTERSECT ALL` in SQL. As standard in SQL, this
function
+ resolves columns by position (not by name).
+
+ .. versionadded:: 2.4.0
Review Comment:
```suggestion
.. versionadded:: 3.4.0
```
##########
python/pyspark/sql/connect/dataframe.py:
##########
@@ -317,7 +319,83 @@ def unionByName(self, other: "DataFrame",
allowMissingColumns: bool = False) ->
if other._plan is None:
raise ValueError("Argument to UnionByName does not contain a valid
plan.")
return DataFrame.withPlan(
- plan.UnionAll(self._plan, other._plan, allowMissingColumns),
session=self._session
+ plan.SetOperation(
+ self._plan, other._plan, "union", is_all=True,
by_name=allowMissingColumns
+ ),
+ session=self._session,
+ )
+
+ def exceptAll(self, other: "DataFrame") -> "DataFrame":
+ """Return a new :class:`DataFrame` containing rows in this
:class:`DataFrame` but
+ not in another :class:`DataFrame` while preserving duplicates.
+
+ This is equivalent to `EXCEPT ALL` in SQL.
+ As standard in SQL, this function resolves columns by position (not by
name).
+
+ .. versionadded:: 2.4.0
Review Comment:
```suggestion
.. versionadded:: 3.4.0
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
