xinrong-databricks commented on a change in pull request #33331:
URL: https://github.com/apache/spark/pull/33331#discussion_r669186589
##########
File path: python/pyspark/pandas/data_type_ops/categorical_ops.py
##########
@@ -64,15 +66,28 @@ def astype(self, index_ops: IndexOpsLike, dtype: Union[str,
type, Dtype]) -> Ind
scol = map_scol.getItem(index_ops.spark.column)
return index_ops._with_new_scol(scol).astype(dtype)
- # TODO(SPARK-35997): Implement comparison operators below
def lt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- raise NotImplementedError("< can not be applied to %s." %
self.pretty_name)
+ _non_equality_comparison_input_check(left, right)
+ return column_op(Column.__lt__)(left, right)
def le(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- raise NotImplementedError("<= can not be applied to %s." %
self.pretty_name)
+ _non_equality_comparison_input_check(left, right)
+ return column_op(Column.__le__)(left, right)
+
+ def gt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
+ _non_equality_comparison_input_check(left, right)
+ return column_op(Column.__gt__)(left, right)
def ge(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- raise NotImplementedError("> can not be applied to %s." %
self.pretty_name)
+ _non_equality_comparison_input_check(left, right)
+ return column_op(Column.__ge__)(left, right)
- def gt(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- raise NotImplementedError(">= can not be applied to %s." %
self.pretty_name)
+
+def _non_equality_comparison_input_check(left: IndexOpsLike, right: Any) ->
None:
+ if not left.dtype.ordered:
+ raise TypeError("Unordered Categoricals can only compare equality or
not.")
+ if isinstance(right, IndexOpsMixin) and isinstance(right.dtype,
CategoricalDtype):
+ if hash(left.dtype) != hash(right.dtype):
Review comment:
Followed pandas comparison
[here](https://github.com/pandas-dev/pandas/blob/master/pandas/core/arrays/categorical.py#L144)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
