xinrong-databricks commented on a change in pull request #33506:
URL: https://github.com/apache/spark/pull/33506#discussion_r676860245
##########
File path: python/pyspark/pandas/categorical.py
##########
@@ -593,12 +595,107 @@ def reorder_categories(
def set_categories(
self,
- new_categories: pd.Index,
+ new_categories: Union[pd.Index, List],
ordered: bool = None,
rename: bool = False,
inplace: bool = False,
- ) -> "ps.Series":
- raise NotImplementedError()
+ ) -> Optional["ps.Series"]:
+ """
+ Set the categories to the specified new_categories.
+
+ `new_categories` can include new categories (which will result in
+ unused categories) or remove old categories (which results in values
+ set to NaN). If `rename==True`, the categories will simple be renamed
+ (less or more items than in old categories will result in values set to
+ NaN or in unused categories respectively).
+
+ This method can be used to perform more than one action of adding,
+ removing, and reordering simultaneously and is therefore faster than
+ performing the individual steps via the more specialised methods.
+
+ On the other hand this methods does not do checks (e.g., whether the
+ old categories are included in the new categories on a reorder), which
+ can result in surprising changes, for example when using special string
+ dtypes, which does not considers a S1 string equal to a single char
+ python string.
+
+ Parameters
+ ----------
+ new_categories : Index-like
+ The categories in new order.
+ ordered : bool, default False
+ Whether or not the categorical is treated as a ordered categorical.
+ If not given, do not change the ordered information.
+ rename : bool, default False
+ Whether or not the new_categories should be considered as a rename
+ of the old categories or as reordered categories.
+ inplace : bool, default False
+ Whether or not to reorder the categories in-place or return a copy
+ of this categorical with reordered categories.
+
+ Returns
+ -------
+ Series with reordered categories or None if inplace.
+
+ Raises
+ ------
+ ValueError
+ If new_categories does not validate as categories
+
+ See Also
+ --------
+ rename_categories : Rename categories.
+ reorder_categories : Reorder categories.
+ add_categories : Add new categories.
+ remove_categories : Remove the specified categories.
+ remove_unused_categories : Remove categories which are not used.
+ """
+ from pyspark.pandas.frame import DataFrame
+
+ if not is_list_like(new_categories):
+ raise TypeError(
+ "Parameter 'new_categories' must be list-like, was
'{}'".format(new_categories)
+ )
+
+ if ordered is None:
+ ordered = self.ordered
+
+ new_dtype = CategoricalDtype(new_categories, ordered=ordered)
+ scol = self._data.spark.column
+
+ if rename:
+ new_scol = (
+ F.when(scol >= len(new_categories), -1)
+ .otherwise(scol)
+ .alias(name_like_string(self._data._column_label))
Review comment:
Makes sense, updated.
##########
File path: python/pyspark/pandas/categorical.py
##########
@@ -593,12 +595,107 @@ def reorder_categories(
def set_categories(
self,
- new_categories: pd.Index,
+ new_categories: Union[pd.Index, List],
ordered: bool = None,
rename: bool = False,
inplace: bool = False,
- ) -> "ps.Series":
- raise NotImplementedError()
+ ) -> Optional["ps.Series"]:
+ """
+ Set the categories to the specified new_categories.
+
+ `new_categories` can include new categories (which will result in
+ unused categories) or remove old categories (which results in values
+ set to NaN). If `rename==True`, the categories will simple be renamed
+ (less or more items than in old categories will result in values set to
+ NaN or in unused categories respectively).
+
+ This method can be used to perform more than one action of adding,
+ removing, and reordering simultaneously and is therefore faster than
+ performing the individual steps via the more specialised methods.
+
+ On the other hand this methods does not do checks (e.g., whether the
+ old categories are included in the new categories on a reorder), which
+ can result in surprising changes, for example when using special string
+ dtypes, which does not considers a S1 string equal to a single char
+ python string.
+
+ Parameters
+ ----------
+ new_categories : Index-like
+ The categories in new order.
+ ordered : bool, default False
+ Whether or not the categorical is treated as a ordered categorical.
+ If not given, do not change the ordered information.
+ rename : bool, default False
+ Whether or not the new_categories should be considered as a rename
+ of the old categories or as reordered categories.
+ inplace : bool, default False
+ Whether or not to reorder the categories in-place or return a copy
+ of this categorical with reordered categories.
+
+ Returns
+ -------
+ Series with reordered categories or None if inplace.
+
+ Raises
+ ------
+ ValueError
+ If new_categories does not validate as categories
+
+ See Also
+ --------
+ rename_categories : Rename categories.
+ reorder_categories : Reorder categories.
+ add_categories : Add new categories.
+ remove_categories : Remove the specified categories.
+ remove_unused_categories : Remove categories which are not used.
+ """
Review comment:
Certainly, added.
##########
File path: python/pyspark/pandas/indexes/category.py
##########
@@ -465,6 +465,70 @@ def rename_categories(
self.name
)
+ def set_categories(
+ self,
+ new_categories: Union[pd.Index, List],
+ ordered: bool = None,
+ rename: bool = False,
+ inplace: bool = False,
+ ) -> Optional["CategoricalIndex"]:
+ """
+ Set the categories to the specified new_categories.
+
+ `new_categories` can include new categories (which will result in
+ unused categories) or remove old categories (which results in values
+ set to NaN). If `rename==True`, the categories will simple be renamed
+ (less or more items than in old categories will result in values set to
+ NaN or in unused categories respectively).
+
+ This method can be used to perform more than one action of adding,
+ removing, and reordering simultaneously and is therefore faster than
+ performing the individual steps via the more specialised methods.
+
+ On the other hand this methods does not do checks (e.g., whether the
+ old categories are included in the new categories on a reorder), which
+ can result in surprising changes, for example when using special string
+ dtypes, which does not considers a S1 string equal to a single char
+ python string.
+
+ Parameters
+ ----------
+ new_categories : Index-like
+ The categories in new order.
+ ordered : bool, default False
+ Whether or not the categorical is treated as a ordered categorical.
+ If not given, do not change the ordered information.
+ rename : bool, default False
+ Whether or not the new_categories should be considered as a rename
+ of the old categories or as reordered categories.
+ inplace : bool, default False
+ Whether or not to reorder the categories in-place or return a copy
+ of this categorical with reordered categories.
+
+ Returns
+ -------
+ CategoricalIndex with reordered categories or None if inplace.
+
+ Raises
+ ------
+ ValueError
+ If new_categories does not validate as categories
+
+ See Also
+ --------
+ rename_categories : Rename categories.
+ reorder_categories : Reorder categories.
+ add_categories : Add new categories.
+ remove_categories : Remove the specified categories.
+ remove_unused_categories : Remove categories which are not used.
+ """
Review comment:
Added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
