xinrong-databricks commented on a change in pull request #36006:
URL: https://github.com/apache/spark/pull/36006#discussion_r839858561
##########
File path: python/pyspark/pandas/indexes/base.py
##########
@@ -898,25 +905,19 @@ def drop_duplicates(self) -> "Index":
Examples
--------
- Generate an pandas.Index with duplicate values.
+ Generate an Index with duplicate values.
>>> idx = ps.Index(['lama', 'cow', 'lama', 'beetle', 'lama', 'hippo'])
>>> idx.drop_duplicates().sort_values()
Index(['beetle', 'cow', 'hippo', 'lama'], dtype='object')
"""
- sdf = self._internal.spark_frame.select(
- self._internal.index_spark_columns
- ).drop_duplicates()
- internal = InternalFrame(
- spark_frame=sdf,
- index_spark_columns=[
- scol_for(sdf, col) for col in
self._internal.index_spark_column_names
- ],
- index_names=self._internal.index_names,
- index_fields=self._internal.index_fields,
- )
- return DataFrame(internal).index
+ with ps.option_context("compute.default_index_type", "distributed"):
+ # The attached index caused by `reset_index` below is used for
sorting only,
+ # and it will be dropped soon,
+ # so we enforce “distributed” default index type
+ psser = self.to_series().reset_index(drop=True)
+ return Index(psser.drop_duplicates(keep=keep).sort_index())
Review comment:
Agree, thank you @itholic !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
