This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b7a5543 [SPARK-36396][PYTHON][FOLLOWUP] Fix test with extensions
dtype when pandas version < 1.2
b7a5543 is described below
commit b7a55435c9f69f3d1e7a4f1967a44a54c4d5d050
Author: dch nguyen <[email protected]>
AuthorDate: Thu Dec 2 17:46:14 2021 +0900
[SPARK-36396][PYTHON][FOLLOWUP] Fix test with extensions dtype when pandas
version < 1.2
### What changes were proposed in this pull request?
Fix test of `pd.Dataframe.cov` with extensions dtype when pandas version <
1.2
### Why are the changes needed?
Pass test of `ps.Dataframe.cov` with pandas version < 1.2
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests and Manual test
Closes #34778 from dchvn/SPARK-36396-FU.
Authored-by: dch nguyen <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/pandas/tests/test_dataframe.py | 39 +++++++++++++++++++++++----
1 file changed, 34 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/pandas/tests/test_dataframe.py
b/python/pyspark/pandas/tests/test_dataframe.py
index d12a084..c2ae4da 100644
--- a/python/pyspark/pandas/tests/test_dataframe.py
+++ b/python/pyspark/pandas/tests/test_dataframe.py
@@ -5870,8 +5870,12 @@ class DataFrameTest(PandasOnSparkTestCase, SQLTestUtils):
self.assert_eq(pdf.cov(min_periods=5), psdf.cov(min_periods=5))
# extension dtype
- numeric_dtypes = ["Int8", "Int16", "Int32", "Int64", "Float32",
"Float64", "float"]
- boolean_dtypes = ["boolean", "bool"]
+ if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+ numeric_dtypes = ["Int8", "Int16", "Int32", "Int64", "Float32",
"Float64", "float"]
+ boolean_dtypes = ["boolean", "bool"]
+ else:
+ numeric_dtypes = ["Int8", "Int16", "Int32", "Int64", "float"]
+ boolean_dtypes = ["boolean", "bool"]
sers = [pd.Series([1, 2, 3, None], dtype=dtype) for dtype in
numeric_dtypes]
sers += [pd.Series([True, False, True, None], dtype=dtype) for dtype
in boolean_dtypes]
@@ -5881,9 +5885,34 @@ class DataFrameTest(PandasOnSparkTestCase, SQLTestUtils):
pdf.columns = [dtype for dtype in numeric_dtypes + boolean_dtypes] +
["decimal"]
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.cov(), psdf.cov(), almost=True)
- self.assert_eq(pdf.cov(min_periods=3), psdf.cov(min_periods=3),
almost=True)
- self.assert_eq(pdf.cov(min_periods=4), psdf.cov(min_periods=4))
+ if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+ self.assert_eq(pdf.cov(), psdf.cov(), almost=True)
+ self.assert_eq(pdf.cov(min_periods=3), psdf.cov(min_periods=3),
almost=True)
+ self.assert_eq(pdf.cov(min_periods=4), psdf.cov(min_periods=4))
+ else:
+ test_types = [
+ "Int8",
+ "Int16",
+ "Int32",
+ "Int64",
+ "float",
+ "boolean",
+ "bool",
+ ]
+ expected = pd.DataFrame(
+ data=[
+ [1.0, 1.0, 1.0, 1.0, 1.0, 0.0000000, 0.0000000],
+ [1.0, 1.0, 1.0, 1.0, 1.0, 0.0000000, 0.0000000],
+ [1.0, 1.0, 1.0, 1.0, 1.0, 0.0000000, 0.0000000],
+ [1.0, 1.0, 1.0, 1.0, 1.0, 0.0000000, 0.0000000],
+ [1.0, 1.0, 1.0, 1.0, 1.0, 0.0000000, 0.0000000],
+ [0.0, 0.0, 0.0, 0.0, 0.0, 0.3333333, 0.3333333],
+ [0.0, 0.0, 0.0, 0.0, 0.0, 0.3333333, 0.3333333],
+ ],
+ index=test_types,
+ columns=test_types,
+ )
+ self.assert_eq(expected, psdf.cov(), almost=True)
# string column
pdf = pd.DataFrame(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
