grundprinzip commented on code in PR #38819:
URL: https://github.com/apache/spark/pull/38819#discussion_r1033669941
##########
python/pyspark/sql/connect/dataframe.py:
##########
@@ -727,6 +727,77 @@ def fillna(
session=self._session,
)
+ def dropna(
+ self,
+ how: str = "any",
+ thresh: Optional[int] = None,
+ subset: Optional[Union[str, Tuple[str, ...], List[str]]] = None,
+ ) -> "DataFrame":
+ """Returns a new :class:`DataFrame` omitting rows with null values.
+ :func:`DataFrame.dropna` and :func:`DataFrameNaFunctions.drop` are
aliases of each other.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ how : str, optional
+ 'any' or 'all'.
+ If 'any', drop a row if it contains any nulls.
+ If 'all', drop a row only if all its values are null.
+ thresh: int, optional
+ default None
+ If specified, drop rows that have less than `thresh` non-null
values.
+ This overwrites the `how` parameter.
+ subset : str, tuple or list, optional
+ optional list of column names to consider.
+
+ Returns
+ -------
+ :class:`DataFrame`
+ DataFrame with null only rows excluded.
+ """
+ min_non_nulls: Optional[int] = None
+
+ if how is not None:
+ if not isinstance(how, str):
+ raise TypeError(f"how should be a str, but got
{type(how).__name__}")
+ if how == "all":
+ min_non_nulls = 1
+ elif how == "any":
+ min_non_nulls = None
+ else:
+ raise ValueError("how ('" + how + "') should be 'any' or
'all'")
+
+ if thresh is not None:
+ if not isinstance(thresh, int):
+ raise TypeError(f"thresh should be a int, but got
{type(thresh).__name__}")
+
+ # 'thresh' overwrites 'how'
+ min_non_nulls = thresh
+
+ _cols: List[str] = []
+ if subset is not None:
+ if isinstance(subset, str):
+ _cols = [subset]
+ elif isinstance(subset, (tuple, list)):
+ for c in subset:
+ if not isinstance(c, str):
+ raise TypeError(
+ f"cols should be a str, tuple[str] or list[str], "
+ f"but got {type(c).__name__}"
+ )
+ _cols = list(subset)
+ else:
+ raise TypeError(
+ f"cols should be a str, tuple[str] or list[str], "
+ f"but got {type(subset).__name__}"
+ )
+
+ return DataFrame.withPlan(
+ plan.NADrop(child=self._plan, cols=_cols,
min_non_nulls=min_non_nulls),
+ session=self._session,
Review Comment:
this should probably `self._session.client` now.
##########
python/pyspark/sql/tests/connect/test_connect_basic.py:
##########
@@ -472,6 +472,46 @@ def test_fill_na(self):
self.spark.sql(query).na.fill({"a": True, "b": 2}).toPandas(),
)
+ def test_drop_na(self):
+ # SPARK-41148: Test drop na
+ query = """
+ SELECT * FROM VALUES
+ (false, 1, NULL), (false, NULL, 2.0), (NULL, 3, 3.0)
+ AS tab(a, b, c)
+ """
+ # +-----+----+----+
+ # | a| b| c|
+ # +-----+----+----+
+ # |false| 1|null|
+ # |false|null| 2.0|
+ # | null| 3| 3.0|
+ # +-----+----+----+
+
+ self.assertTrue(
Review Comment:
is `self.assert_eq` not applicable here? If we consistently hit issues where
we can't use the Pandas equality helper let's figure out how to improve.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
