This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 5dda2fa4925 [SPARK-44948][DOCS][TESTS][PYTHON] Update document & test related to `Int64Index` 5dda2fa4925 is described below commit 5dda2fa492551653b1ecf32481da1e721b18c89d Author: itholic <haejoon....@databricks.com> AuthorDate: Fri Aug 25 09:15:07 2023 +0900 [SPARK-44948][DOCS][TESTS][PYTHON] Update document & test related to `Int64Index` ### What changes were proposed in this pull request? This PR proposes to update the document & tests related to `ps.Int64Index`, which is removed from https://github.com/apache/spark/pull/42267. ### Why are the changes needed? To maintain consistency between features, documentation and tests ### Does this PR introduce _any_ user-facing change? No, but it's related to user-facing documentation. ### How was this patch tested? Manually tested, and the existing CI should pass. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #42662 from itholic/SPARK-44948. Authored-by: itholic <haejoon....@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- python/docs/source/getting_started/quickstart_ps.ipynb | 2 +- python/docs/source/user_guide/pandas_on_spark/options.rst | 6 +++--- python/pyspark/pandas/tests/indexes/test_base.py | 6 +----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/python/docs/source/getting_started/quickstart_ps.ipynb b/python/docs/source/getting_started/quickstart_ps.ipynb index dc47bdfa2c6..02884a00859 100644 --- a/python/docs/source/getting_started/quickstart_ps.ipynb +++ b/python/docs/source/getting_started/quickstart_ps.ipynb @@ -785,7 +785,7 @@ { "data": { "text/plain": [ - "Int64Index([0, 1, 2, 3, 4, 5], dtype='int64')" + "Index([0, 1, 2, 3, 4, 5], dtype='int64')" ] }, "execution_count": 20, diff --git a/python/docs/source/user_guide/pandas_on_spark/options.rst b/python/docs/source/user_guide/pandas_on_spark/options.rst index 92b572b9d80..774a0e23f87 100644 --- a/python/docs/source/user_guide/pandas_on_spark/options.rst +++ b/python/docs/source/user_guide/pandas_on_spark/options.rst @@ -169,7 +169,7 @@ This index type should be avoided when the data is large. See the example below: >>> psdf = ps.range(3) >>> ps.reset_option('compute.default_index_type') >>> psdf.index - Int64Index([0, 1, 2], dtype='int64') + Index([0, 1, 2], dtype='int64') This is conceptually equivalent to the PySpark example as below: @@ -195,7 +195,7 @@ index has to be used. See the example below: >>> psdf = ps.range(3) >>> ps.reset_option('compute.default_index_type') >>> psdf.index - Int64Index([0, 1, 2], dtype='int64') + Index([0, 1, 2], dtype='int64') This is conceptually equivalent to the PySpark example as below: @@ -219,7 +219,7 @@ have any penalty compared to other index types. See the example below: >>> psdf = ps.range(3) >>> ps.reset_option('compute.default_index_type') >>> psdf.index - Int64Index([25769803776, 60129542144, 94489280512], dtype='int64') + Index([25769803776, 60129542144, 94489280512], dtype='int64') This is conceptually equivalent to the PySpark example as below: diff --git a/python/pyspark/pandas/tests/indexes/test_base.py b/python/pyspark/pandas/tests/indexes/test_base.py index eff8c19c36d..bf2ad0a630b 100644 --- a/python/pyspark/pandas/tests/indexes/test_base.py +++ b/python/pyspark/pandas/tests/indexes/test_base.py @@ -59,11 +59,7 @@ class IndexesTestsMixin: ]: psdf = ps.from_pandas(pdf) self.assert_eq(psdf.index, pdf.index) - # Int64Index is removed from pandas 2.0.0, so we should compare the dtype itself. - if LooseVersion(pd.__version__) >= LooseVersion("2.0.0"): - self.assert_eq(psdf.index.dtype, pdf.index.dtype) - else: - self.assert_eq(type(psdf.index).__name__, type(pdf.index).__name__) + self.assert_eq(psdf.index.dtype, pdf.index.dtype) self.assert_eq(ps.Index([])._summary(), "Index: 0 entries") with self.assertRaisesRegexp(ValueError, "The truth value of a Index is ambiguous."): --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org